From 32983e90d181cf887d81bec8aae5a3b91b25813b Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 1 Jun 2023 12:58:56 +0200 Subject: [PATCH 01/12] change to the model of the Relation -> flatten: remove the node and add source, sourceType, target, targetType. Adding indicators at the level of Projects and Datasources. Removing the prefix from the identifier of the entities --- .../dhp/oa/model/graph/Datasource.java | 12 + .../dnetlib/dhp/oa/model/graph/Project.java | 5 + .../dnetlib/dhp/oa/model/graph/Relation.java | 49 +- .../dhp/oa/graph/dump/ResultMapper.java | 677 +++++++++--------- .../eu/dnetlib/dhp/oa/graph/dump/Utils.java | 76 +- .../dhp/oa/graph/dump/complete/Extractor.java | 16 +- .../dhp/oa/graph/dump/complete/Process.java | 13 +- .../dump/complete/SparkCollectAndSave.java | 2 +- .../dump/complete/SparkDumpEntitiesJob.java | 5 +- .../dump/complete/SparkDumpRelationJob.java | 14 +- .../complete/SparkOrganizationRelation.java | 9 +- .../dump/subset/SparkSelectValidRelation.java | 2 +- .../SparkSelectValidRelationContext.java | 6 +- .../dhp/oa/graph/dump/DumpJobTest.java | 14 +- .../dump/complete/CreateRelationTest.java | 24 +- .../graph/dump/complete/DumpRelationTest.java | 8 +- .../ExtractRelationFromEntityTest.java | 40 +- .../oa/graph/dump/subset/DumpSubsetTest.java | 23 +- .../dump/subset/dump/community_infrastructure | 12 +- .../dhp/oa/graph/dump/subset/dump/datasource | 8 +- .../oa/graph/dump/subset/dump/organization | 6 +- .../dhp/oa/graph/dump/subset/dump/project | 6 +- .../dhp/oa/graph/dump/subset/dump/publication | 30 +- .../dump/subset/working/relation/context | 50 +- .../dump/subset/working/relation/contextOrg | 12 +- pom.xml | 2 +- 26 files changed, 613 insertions(+), 508 deletions(-) diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Datasource.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Datasource.java index 7984f87..7154287 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Datasource.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Datasource.java @@ -7,6 +7,7 @@ import java.util.List; import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema; import eu.dnetlib.dhp.oa.model.Container; +import eu.dnetlib.dhp.oa.model.Indicator; /** * To store information about the datasource OpenAIRE collects information from. It contains the following parameters: - @@ -128,6 +129,17 @@ public class Datasource implements Serializable { @JsonSchema(description = "Information about the journal, if this data source is of type Journal.") private Container journal; // issn etc del Journal + @JsonSchema(description = "Indicators computed for this Datasource, for example UsageCount ones") + private Indicator indicators; + + public Indicator getIndicators() { + return indicators; + } + + public void setIndicators(Indicator indicators) { + this.indicators = indicators; + } + public String getId() { return id; } diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java index 67aa076..6480c87 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java @@ -6,6 +6,8 @@ import java.util.List; import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema; +import eu.dnetlib.dhp.oa.model.Indicator; + /** * This is the class representing the Project in the model used for the dumps of the whole graph. At the moment the dump * of the Projects differs from the other dumps because we do not create relations between Funders (Organization) and @@ -68,6 +70,9 @@ public class Project implements Serializable { @JsonSchema(description = "The h2020 programme funding the project") private List h2020programme; + @JsonSchema(description = "Indicators computed for this project, for example UsageCount ones") + private Indicator indicators; + public String getId() { return id; } diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Relation.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Relation.java index 9f3832d..ceb0339 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Relation.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Relation.java @@ -15,11 +15,17 @@ import eu.dnetlib.dhp.oa.model.Provenance; * provenance of the relation */ public class Relation implements Serializable { - @JsonSchema(description = "The node source in the relation") - private Node source; + @JsonSchema(description = "The identifier of the source in the relation") + private String source; - @JsonSchema(description = "The node target in the relation") - private Node target; + @JsonSchema(description = "The entity type of the source in the relation") + private String sourceType; + + @JsonSchema(description = "The identifier of the target in the relation") + private String target; + + @JsonSchema(description = "The entity type of the target in the relation") + private String targetType; @JsonSchema(description = "To represent the semantics of a relation between two entities") private RelType reltype; @@ -34,22 +40,42 @@ public class Relation implements Serializable { @JsonSchema(description = "The date when the relation was collected from OpenAIRE") private String validationDate; - public Node getSource() { + public String getSource() { return source; } - public void setSource(Node source) { + public void setSource(String source) { this.source = source; } - public Node getTarget() { + public String getSourceType() { + return sourceType; + } + + public void setSourceType(String sourceType) { + this.sourceType = sourceType; + } + + public String getTarget() { return target; } - public void setTarget(Node target) { + public void setTarget(String target) { this.target = target; } + public String getTargetType() { + return targetType; + } + + public void setTargetType(String targetType) { + this.targetType = targetType; + } + + public boolean isValidated() { + return validated; + } + public RelType getReltype() { return reltype; } @@ -85,13 +111,16 @@ public class Relation implements Serializable { @Override public int hashCode() { - return Objects.hash(source.getId(), target.getId(), reltype.getType() + ":" + reltype.getName()); + return Objects.hash(source, target, reltype.getType() + ":" + reltype.getName()); } - public static Relation newInstance(Node source, Node target, RelType reltype, Provenance provenance) { + public static Relation newInstance(String source, String sourceType, String target, String targetType, + RelType reltype, Provenance provenance) { Relation relation = new Relation(); relation.source = source; + relation.sourceType = sourceType; relation.target = target; + relation.targetType = targetType; relation.reltype = reltype; relation.provenance = provenance; return relation; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java index 9414cda..dd01252 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java @@ -8,8 +8,7 @@ import java.util.*; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.jetbrains.annotations.NotNull; import eu.dnetlib.dhp.oa.graph.dump.exceptions.CardinalityTooHighException; import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException; @@ -21,6 +20,7 @@ import eu.dnetlib.dhp.oa.model.Indicator; import eu.dnetlib.dhp.oa.model.Instance; import eu.dnetlib.dhp.oa.model.OpenAccessRoute; import eu.dnetlib.dhp.oa.model.Result; +import eu.dnetlib.dhp.oa.model.Subject; import eu.dnetlib.dhp.oa.model.community.CfHbKeyValue; import eu.dnetlib.dhp.oa.model.community.CommunityInstance; import eu.dnetlib.dhp.oa.model.community.CommunityResult; @@ -30,7 +30,6 @@ import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; public class ResultMapper implements Serializable { - private static final Logger log = LoggerFactory.getLogger(ResultMapper.class); public static Result map( E in, Map communityMap, String dumpType) @@ -48,279 +47,32 @@ public class ResultMapper implements Serializable { if (ort.isPresent()) { try { - addTypeSpecificInformation(out, input, ort); - - Optional - .ofNullable(input.getAuthor()) - .ifPresent( - ats -> out.setAuthor(ats.stream().map(ResultMapper::getAuthor).collect(Collectors.toList()))); - - // I do not map Access Right UNKNOWN or OTHER - - Optional oar = Optional.ofNullable(input.getBestaccessright()); - if (oar.isPresent() && Constants.ACCESS_RIGHTS_COAR_MAP.containsKey(oar.get().getClassid())) { - String code = Constants.ACCESS_RIGHTS_COAR_MAP.get(oar.get().getClassid()); - out - .setBestaccessright( - - BestAccessRight - .newInstance( - code, - Constants.COAR_CODE_LABEL_MAP.get(code), - Constants.COAR_ACCESS_RIGHT_SCHEMA)); - } - - final List contributorList = new ArrayList<>(); - Optional - .ofNullable(input.getContributor()) - .ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue()))); - out.setContributor(contributorList); - - Optional - .ofNullable(input.getCountry()) - .ifPresent( - value -> out - .setCountry( - value - .stream() - .map( - c -> { - if (c.getClassid().equals((ModelConstants.UNKNOWN))) { - return null; - } - ResultCountry country = new ResultCountry(); - country.setCode(c.getClassid()); - country.setLabel(c.getClassname()); - Optional - .ofNullable(c.getDataInfo()) - .ifPresent( - provenance -> country - .setProvenance( - Provenance - .newInstance( - provenance - .getProvenanceaction() - .getClassname(), - c.getDataInfo().getTrust()))); - return country; - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()))); - - final List coverageList = new ArrayList<>(); - Optional - .ofNullable(input.getCoverage()) - .ifPresent(value -> value.stream().forEach(c -> coverageList.add(c.getValue()))); - out.setCoverage(coverageList); - + addTypeSpecificInformation(out, input, ort.get()); + mapAuthor(out, input); + mapAccessRight(out, input); + mapContributor(out, input); + mapCountry(out, input); + mapCoverage(out, input); out.setDateofcollection(input.getDateofcollection()); - - final List descriptionList = new ArrayList<>(); - Optional - .ofNullable(input.getDescription()) - .ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue()))); - out.setDescription(descriptionList); - Optional> oStr = Optional.ofNullable(input.getEmbargoenddate()); - if (oStr.isPresent()) { - out.setEmbargoenddate(oStr.get().getValue()); - } - - final List formatList = new ArrayList<>(); - Optional - .ofNullable(input.getFormat()) - .ifPresent(value -> value.stream().forEach(f -> formatList.add(f.getValue()))); - out.setFormat(formatList); - out.setId(input.getId()); - out.setOriginalId(new ArrayList<>()); - Optional - .ofNullable(input.getOriginalId()) - .ifPresent( - v -> out - .setOriginalId( - input - .getOriginalId() - .stream() - .filter(s -> !s.startsWith("50|")) - .collect(Collectors.toList()))); - - Optional> oInst = Optional - .ofNullable(input.getInstance()); - - if (oInst.isPresent()) { - if (Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) { - ((GraphResult) out) - .setInstance( - oInst.get().stream().map(ResultMapper::getGraphInstance).collect(Collectors.toList())); - } else { - ((CommunityResult) out) - .setInstance( - oInst - .get() - .stream() - .map(ResultMapper::getCommunityInstance) - .collect(Collectors.toList())); - } - } - - Optional oL = Optional.ofNullable(input.getLanguage()); - if (oL.isPresent()) { - eu.dnetlib.dhp.schema.oaf.Qualifier language = oL.get(); - out.setLanguage(Language.newInstance(language.getClassid(), language.getClassname())); - } - Optional oLong = Optional.ofNullable(input.getLastupdatetimestamp()); - if (oLong.isPresent()) { - out.setLastupdatetimestamp(oLong.get()); - } - Optional> otitle = Optional.ofNullable(input.getTitle()); - if (otitle.isPresent()) { - List iTitle = otitle - .get() - .stream() - .filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title")) - .collect(Collectors.toList()); - if (!iTitle.isEmpty()) { - out.setMaintitle(iTitle.get(0).getValue()); - } - - iTitle = otitle - .get() - .stream() - .filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle")) - .collect(Collectors.toList()); - if (!iTitle.isEmpty()) { - out.setSubtitle(iTitle.get(0).getValue()); - } - - } - - Optional - .ofNullable(input.getPid()) - .ifPresent( - value -> out - .setPid( - value - .stream() - .map( - p -> ResultPid - .newInstance(p.getQualifier().getClassid(), p.getValue())) - .collect(Collectors.toList()))); - - oStr = Optional.ofNullable(input.getDateofacceptance()); - if (oStr.isPresent()) { - out.setPublicationdate(oStr.get().getValue()); - } - oStr = Optional.ofNullable(input.getPublisher()); - if (oStr.isPresent()) { - out.setPublisher(oStr.get().getValue()); - } - - Optional - .ofNullable(input.getSource()) - .ifPresent( - value -> out.setSource(value.stream().map(Field::getValue).collect(Collectors.toList()))); - - List subjectList = new ArrayList<>(); - Optional - .ofNullable(input.getSubject()) - .ifPresent( - value -> value - .stream() - .filter( - s -> !((s.getQualifier().getClassid().equalsIgnoreCase("fos") && - Optional.ofNullable(s.getDataInfo()).isPresent() - && Optional.ofNullable(s.getDataInfo().getProvenanceaction()).isPresent() && - s.getDataInfo().getProvenanceaction().getClassid().equalsIgnoreCase("subject:fos")) - || - (s.getQualifier().getClassid().equalsIgnoreCase("sdg") && - Optional.ofNullable(s.getDataInfo()).isPresent() - && Optional.ofNullable(s.getDataInfo().getProvenanceaction()).isPresent() && - s - .getDataInfo() - .getProvenanceaction() - .getClassid() - .equalsIgnoreCase("subject:sdg")))) - .forEach(s -> subjectList.add(getSubject(s)))); - - out.setSubjects(subjectList); - + mapDescription(out, input); + mapEmbargo(out, input); + mapFormat(out, input); + out.setId(input.getId().substring(3)); + mapOriginalId(out, input); + mapInstance(dumpType, out, input); + mapLanguage(out, input); + mapLastUpdateTimestamp(out, input); + mapTitle(out, input); + mapPid(out, input); + mapDateOfAcceptance(out, input); + mapPublisher(out, input); + mapSource(out, input); + mapSubject(out, input); out.setType(input.getResulttype().getClassid()); - - if (Optional.ofNullable(input.getMeasures()).isPresent() && input.getMeasures().size() > 0) { - - out.setIndicators(getIndicator(input.getMeasures())); - } - + mapMeasure(out, input); if (!Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) { - ((CommunityResult) out) - .setCollectedfrom( - input - .getCollectedfrom() - .stream() - .map(cf -> CfHbKeyValue.newInstance(cf.getKey(), cf.getValue())) - .collect(Collectors.toList())); - - Set communities = communityMap.keySet(); - List contextList = Optional - .ofNullable( - input - .getContext()) - .map( - value -> value - .stream() - .map(c -> { - String communityId = c.getId(); - if (communityId.contains("::")) { - communityId = communityId.substring(0, communityId.indexOf("::")); - } - if (communities.contains(communityId)) { - Context context = new Context(); - context.setCode(communityId); - context.setLabel(communityMap.get(communityId)); - Optional> dataInfo = Optional.ofNullable(c.getDataInfo()); - if (dataInfo.isPresent()) { - List provenance = new ArrayList<>(); - provenance - .addAll( - dataInfo - .get() - .stream() - .map( - di -> Optional - .ofNullable(di.getProvenanceaction()) - .map( - provenanceaction -> Provenance - .newInstance( - provenanceaction.getClassname(), - di.getTrust())) - .orElse(null)) - .filter(Objects::nonNull) - .collect(Collectors.toSet())); - - try { - context.setProvenance(getUniqueProvenance(provenance)); - } catch (NoAvailableEntityTypeException e) { - e.printStackTrace(); - } - } - return context; - } - return null; - }) - .filter(Objects::nonNull) - .collect(Collectors.toList())) - .orElse(new ArrayList<>()); - - if (!contextList.isEmpty()) { - Set hashValue = new HashSet<>(); - List remainigContext = new ArrayList<>(); - contextList.forEach(c -> { - if (!hashValue.contains(c.hashCode())) { - remainigContext.add(c); - hashValue.add(c.hashCode()); - } - }); - ((CommunityResult) out).setContext(remainigContext); - } + mapCollectedfrom((CommunityResult) out, input); + mapContext(communityMap, (CommunityResult) out, input); } } catch (ClassCastException cce) { return null; @@ -331,82 +83,325 @@ public class ResultMapper implements Serializable { } - private static Indicator getIndicator(List measures) { - UsageCounts uc = null; - ImpactMeasures im = null; - Indicator i = new Indicator(); - for (eu.dnetlib.dhp.schema.oaf.Measure m : measures) { - switch (m.getId()) { - case USAGE_COUNT_DOWNLOADS: - if (uc == null) { - uc = new UsageCounts(); - i.setUsageCounts(uc); - } - uc.setDownloads(m.getUnit().get(0).getValue()); - break; - case USAGE_COUNT_VIEWS: - if (uc == null) { - uc = new UsageCounts(); - i.setUsageCounts(uc); - } - uc.setViews(m.getUnit().get(0).getValue()); - break; - case IMPACT_POPULARITY: - if (im == null) { - im = new ImpactMeasures(); - i.setImpactMeasures(im); - } - im.setPopularity(getScore(m.getUnit())); - break; - case IMPACT_POPULARITY_ALT: - if (im == null) { - im = new ImpactMeasures(); - i.setImpactMeasures(im); - } - im.setPopularity_alt(getScore(m.getUnit())); - break; - case IMPACT_IMPULSE: - if (im == null) { - im = new ImpactMeasures(); - i.setImpactMeasures(im); - } - im.setImpulse(getScore(m.getUnit())); - break; - case IMPACT_INFLUENCE: - if (im == null) { - im = new ImpactMeasures(); - i.setImpactMeasures(im); - } - im.setInfluence(getScore(m.getUnit())); - break; - case IMPACT_INFLUENCE_ALT: - if (im == null) { - im = new ImpactMeasures(); - i.setImpactMeasures(im); - } - im.setInfluence_alt(getScore(m.getUnit())); - break; - } - } + private static void mapContext(Map communityMap, CommunityResult out, + eu.dnetlib.dhp.schema.oaf.Result input) { + Set communities = communityMap.keySet(); + List contextList = Optional + .ofNullable( + input + .getContext()) + .map( + value -> value + .stream() + .map(c -> { + String communityId = c.getId(); + if (communityId.contains("::")) { + communityId = communityId.substring(0, communityId.indexOf("::")); + } + if (communities.contains(communityId)) { + Context context = new Context(); + context.setCode(communityId); + context.setLabel(communityMap.get(communityId)); + Optional> dataInfo = Optional.ofNullable(c.getDataInfo()); + if (dataInfo.isPresent()) { + List provenance = new ArrayList<>(); + provenance + .addAll( + dataInfo + .get() + .stream() + .map( + di -> Optional + .ofNullable(di.getProvenanceaction()) + .map( + provenanceaction -> Provenance + .newInstance( + provenanceaction.getClassname(), + di.getTrust())) + .orElse(null)) + .filter(Objects::nonNull) + .collect(Collectors.toSet())); - return i; + try { + context.setProvenance(getUniqueProvenance(provenance)); + } catch (NoAvailableEntityTypeException e) { + e.printStackTrace(); + } + } + return context; + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList())) + .orElse(new ArrayList<>()); + + if (!contextList.isEmpty()) { + Set hashValue = new HashSet<>(); + List remainigContext = new ArrayList<>(); + contextList.forEach(c -> { + if (!hashValue.contains(c.hashCode())) { + remainigContext.add(c); + hashValue.add(c.hashCode()); + } + }); + out.setContext(remainigContext); + } } - private static Score getScore(List unit) { - Score s = new Score(); - for (KeyValue u : unit) { - if (u.getKey().equals("score")) { - s.setScore(u.getValue()); + private static void mapCollectedfrom(CommunityResult out, eu.dnetlib.dhp.schema.oaf.Result input) { + out + .setCollectedfrom( + input + .getCollectedfrom() + .stream() + .map(cf -> CfHbKeyValue.newInstance(cf.getKey(), cf.getValue())) + .collect(Collectors.toList())); + } + + private static void mapMeasure(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + if (Optional.ofNullable(input.getMeasures()).isPresent() && input.getMeasures().size() > 0) { + + out.setIndicators(Utils.getIndicator(input.getMeasures())); + } + } + + private static void mapSubject(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + List subjectList = new ArrayList<>(); + Optional + .ofNullable(input.getSubject()) + .ifPresent( + value -> value + .stream() + .filter( + s -> !((s.getQualifier().getClassid().equalsIgnoreCase("fos") && + Optional.ofNullable(s.getDataInfo()).isPresent() + && Optional.ofNullable(s.getDataInfo().getProvenanceaction()).isPresent() && + s.getDataInfo().getProvenanceaction().getClassid().equalsIgnoreCase("subject:fos")) + || + (s.getQualifier().getClassid().equalsIgnoreCase("sdg") && + Optional.ofNullable(s.getDataInfo()).isPresent() + && Optional.ofNullable(s.getDataInfo().getProvenanceaction()).isPresent() && + s + .getDataInfo() + .getProvenanceaction() + .getClassid() + .equalsIgnoreCase("subject:sdg")))) + .forEach(s -> subjectList.add(getSubject(s)))); + + out.setSubjects(subjectList); + } + + private static void mapSource(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional + .ofNullable(input.getSource()) + .ifPresent( + value -> out.setSource(value.stream().map(Field::getValue).collect(Collectors.toList()))); + } + + private static void mapPublisher(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional> oStr; + oStr = Optional.ofNullable(input.getPublisher()); + if (oStr.isPresent()) { + out.setPublisher(oStr.get().getValue()); + } + } + + private static void mapDateOfAcceptance(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional> oStr; + oStr = Optional.ofNullable(input.getDateofacceptance()); + if (oStr.isPresent()) { + out.setPublicationdate(oStr.get().getValue()); + } + } + + private static void mapPid(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional + .ofNullable(input.getPid()) + .ifPresent( + value -> out + .setPid( + value + .stream() + .map( + p -> ResultPid + .newInstance(p.getQualifier().getClassid(), p.getValue())) + .collect(Collectors.toList()))); + } + + private static void mapTitle(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional> otitle = Optional.ofNullable(input.getTitle()); + if (otitle.isPresent()) { + List iTitle = otitle + .get() + .stream() + .filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title")) + .collect(Collectors.toList()); + if (!iTitle.isEmpty()) { + out.setMaintitle(iTitle.get(0).getValue()); + } + + iTitle = otitle + .get() + .stream() + .filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle")) + .collect(Collectors.toList()); + if (!iTitle.isEmpty()) { + out.setSubtitle(iTitle.get(0).getValue()); + } + + } + } + + private static void mapLastUpdateTimestamp(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional oLong = Optional.ofNullable(input.getLastupdatetimestamp()); + if (oLong.isPresent()) { + out.setLastupdatetimestamp(oLong.get()); + } + } + + private static void mapLanguage(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional oL = Optional.ofNullable(input.getLanguage()); + if (oL.isPresent()) { + Qualifier language = oL.get(); + out.setLanguage(Language.newInstance(language.getClassid(), language.getClassname())); + } + } + + private static void mapInstance(String dumpType, Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional> oInst = Optional + .ofNullable(input.getInstance()); + + if (oInst.isPresent()) { + if (DUMPTYPE.COMPLETE.getType().equals(dumpType)) { + ((GraphResult) out) + .setInstance( + oInst.get().stream().map(ResultMapper::getGraphInstance).collect(Collectors.toList())); } else { - s.setClazz(u.getValue()); + ((CommunityResult) out) + .setInstance( + oInst + .get() + .stream() + .map(ResultMapper::getCommunityInstance) + .collect(Collectors.toList())); } } - return s; + } + + private static void mapOriginalId(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + out.setOriginalId(new ArrayList<>()); + Optional + .ofNullable(input.getOriginalId()) + .ifPresent( + v -> out + .setOriginalId( + input + .getOriginalId() + .stream() + .filter(s -> !s.startsWith("50|")) + .collect(Collectors.toList()))); + } + + private static void mapFormat(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + final List formatList = new ArrayList<>(); + Optional + .ofNullable(input.getFormat()) + .ifPresent(value -> value.stream().forEach(f -> formatList.add(f.getValue()))); + out.setFormat(formatList); + } + + private static void mapEmbargo(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional> oStr = Optional.ofNullable(input.getEmbargoenddate()); + if (oStr.isPresent()) { + out.setEmbargoenddate(oStr.get().getValue()); + } + } + + private static void mapDescription(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + final List descriptionList = new ArrayList<>(); + Optional + .ofNullable(input.getDescription()) + .ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue()))); + out.setDescription(descriptionList); + } + + private static void mapCoverage(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + final List coverageList = new ArrayList<>(); + Optional + .ofNullable(input.getCoverage()) + .ifPresent(value -> value.stream().forEach(c -> coverageList.add(c.getValue()))); + out.setCoverage(coverageList); + } + + private static void mapCountry(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional + .ofNullable(input.getCountry()) + .ifPresent( + value -> out + .setCountry( + value + .stream() + .map( + c -> { + if (c.getClassid().equals((ModelConstants.UNKNOWN))) { + return null; + } + ResultCountry country = new ResultCountry(); + country.setCode(c.getClassid()); + country.setLabel(c.getClassname()); + Optional + .ofNullable(c.getDataInfo()) + .ifPresent( + provenance -> country + .setProvenance( + Provenance + .newInstance( + provenance + .getProvenanceaction() + .getClassname(), + c.getDataInfo().getTrust()))); + return country; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()))); + } + + private static void mapContributor(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + final List contributorList = new ArrayList<>(); + Optional + .ofNullable(input.getContributor()) + .ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue()))); + out.setContributor(contributorList); + } + + private static void mapAccessRight(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + // I do not map Access Right UNKNOWN or OTHER + + Optional oar = Optional.ofNullable(input.getBestaccessright()); + if (oar.isPresent() && Constants.ACCESS_RIGHTS_COAR_MAP.containsKey(oar.get().getClassid())) { + String code = Constants.ACCESS_RIGHTS_COAR_MAP.get(oar.get().getClassid()); + out + .setBestaccessright( + + BestAccessRight + .newInstance( + code, + Constants.COAR_CODE_LABEL_MAP.get(code), + Constants.COAR_ACCESS_RIGHT_SCHEMA)); + } + } + + private static void mapAuthor(Result out, eu.dnetlib.dhp.schema.oaf.Result input) { + Optional + .ofNullable(input.getAuthor()) + .ifPresent( + ats -> out.setAuthor(ats.stream().map(ResultMapper::getAuthor).collect(Collectors.toList()))); } private static void addTypeSpecificInformation(Result out, eu.dnetlib.dhp.schema.oaf.Result input, - Optional ort) throws NoAvailableEntityTypeException { - switch (ort.get().getClassid()) { + eu.dnetlib.dhp.schema.oaf.Qualifier ort) throws NoAvailableEntityTypeException { + switch (ort.getClassid()) { case "publication": Optional journal = Optional .ofNullable(((Publication) input).getJournal()); @@ -547,8 +542,6 @@ public class ResultMapper implements Serializable { Constants.COAR_CODE_LABEL_MAP.get(code), Constants.COAR_ACCESS_RIGHT_SCHEMA)); - Optional> mes = Optional.ofNullable(i.getMeasures()); - if (opAr.get().getOpenAccessRoute() != null) { switch (opAr.get().getOpenAccessRoute()) { case hybrid: diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java index 7328ce8..dca42f4 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java @@ -1,9 +1,12 @@ package eu.dnetlib.dhp.oa.graph.dump; +import static eu.dnetlib.dhp.oa.graph.dump.Constants.*; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -12,6 +15,7 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; +import org.jetbrains.annotations.NotNull; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -19,9 +23,15 @@ import com.google.gson.Gson; import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.graph.dump.complete.Constants; +import eu.dnetlib.dhp.oa.model.ImpactMeasures; +import eu.dnetlib.dhp.oa.model.Indicator; +import eu.dnetlib.dhp.oa.model.Score; +import eu.dnetlib.dhp.oa.model.UsageCounts; import eu.dnetlib.dhp.oa.model.graph.GraphResult; import eu.dnetlib.dhp.oa.model.graph.Relation; import eu.dnetlib.dhp.oa.model.graph.ResearchCommunity; +import eu.dnetlib.dhp.schema.oaf.KeyValue; +import eu.dnetlib.dhp.schema.oaf.Measure; import eu.dnetlib.dhp.utils.DHPUtils; import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; @@ -119,18 +129,18 @@ public class Utils { return dumpedIds; } - public static Dataset getValidRelations(SparkSession spark, Dataset relations, + public static Dataset getValidRelations(Dataset relations, Dataset entitiesIds) { Dataset> relationSource = relations .map( - (MapFunction>) r -> new Tuple2<>(r.getSource().getId(), r), + (MapFunction>) r -> new Tuple2<>(r.getSource(), r), Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))); Dataset> relJoinSource = relationSource .joinWith(entitiesIds, relationSource.col("_1").equalTo(entitiesIds.col("value"))) .map( (MapFunction, String>, Tuple2>) t2 -> new Tuple2<>( - t2._1()._2().getTarget().getId(), t2._1()._2()), + t2._1()._2().getTarget(), t2._1()._2()), Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))); return relJoinSource @@ -140,4 +150,64 @@ public class Utils { Encoders.bean(Relation.class)); } + public static Indicator getIndicator(List measures) { + Indicator i = new Indicator(); + for (eu.dnetlib.dhp.schema.oaf.Measure m : measures) { + switch (m.getId()) { + case USAGE_COUNT_DOWNLOADS: + getUsageCounts(i).setDownloads(m.getUnit().get(0).getValue()); + break; + case USAGE_COUNT_VIEWS: + getUsageCounts(i).setViews(m.getUnit().get(0).getValue()); + break; + case IMPACT_POPULARITY: + getImpactMeasure(i).setPopularity(getScore(m.getUnit())); + break; + case IMPACT_POPULARITY_ALT: + getImpactMeasure(i).setPopularity_alt(getScore(m.getUnit())); + break; + case IMPACT_IMPULSE: + getImpactMeasure(i).setImpulse(getScore(m.getUnit())); + break; + case IMPACT_INFLUENCE: + getImpactMeasure(i).setInfluence(getScore(m.getUnit())); + break; + case IMPACT_INFLUENCE_ALT: + getImpactMeasure(i).setInfluence_alt(getScore(m.getUnit())); + break; + default: + break; + } + } + + return i; + } + + @NotNull + private static UsageCounts getUsageCounts(Indicator i) { + if (i.getUsageCounts() == null) { + i.setUsageCounts(new UsageCounts()); + } + return i.getUsageCounts(); + } + + @NotNull + private static ImpactMeasures getImpactMeasure(Indicator i) { + if (i.getImpactMeasures() == null) { + i.setImpactMeasures(new ImpactMeasures()); + } + return i.getImpactMeasures(); + } + + private static Score getScore(List unit) { + Score s = new Score(); + for (KeyValue u : unit) { + if (u.getKey().equals("score")) { + s.setScore(u.getValue()); + } else { + s.setClazz(u.getValue()); + } + } + return s; + } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java index 1f2d1a4..5bcc571 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java @@ -85,7 +85,7 @@ public class Extractor implements Serializable { .orElse(null)) .orElse(null); Relation r = getRelation( - value.getId(), contextId, + value.getId().substring(3), contextId.substring(3), Constants.RESULT_ENTITY, Constants.CONTEXT_ENTITY, ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP, provenance); @@ -95,7 +95,7 @@ public class Extractor implements Serializable { hashCodes.add(r.hashCode()); } r = getRelation( - contextId, value.getId(), + contextId.substring(3), value.getId().substring(3), Constants.CONTEXT_ENTITY, Constants.RESULT_ENTITY, ModelConstants.IS_RELATED_TO, @@ -164,8 +164,8 @@ public class Extractor implements Serializable { eu.dnetlib.dhp.oa.graph.dump.Constants.HARVESTED, eu.dnetlib.dhp.oa.graph.dump.Constants.DEFAULT_TRUST)); Relation r = getRelation( - value.getId(), - cf.getKey(), Constants.RESULT_ENTITY, Constants.DATASOURCE_ENTITY, + value.getId().substring(3), + cf.getKey().substring(3), Constants.RESULT_ENTITY, Constants.DATASOURCE_ENTITY, resultDatasource, ModelConstants.PROVISION, provenance); if (!hashCodes.contains(r.hashCode())) { @@ -175,7 +175,7 @@ public class Extractor implements Serializable { } r = getRelation( - cf.getKey(), value.getId(), + cf.getKey().substring(3), value.getId().substring(3), Constants.DATASOURCE_ENTITY, Constants.RESULT_ENTITY, datasourceResult, ModelConstants.PROVISION, provenance); @@ -191,8 +191,10 @@ public class Extractor implements Serializable { private static Relation getRelation(String source, String target, String sourceType, String targetType, String relName, String relType, Provenance provenance) { Relation r = new Relation(); - r.setSource(Node.newInstance(source, sourceType)); - r.setTarget(Node.newInstance(target, targetType)); + r.setSource(source); + r.setSourceType(sourceType); + r.setTarget(target); + r.setTargetType(targetType); r.setReltype(RelType.newInstance(relName, relType)); r.setProvenance(provenance); return r; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java index 07511a9..55390c6 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java @@ -63,10 +63,9 @@ public class Process implements Serializable { .add( Relation .newInstance( - Node - .newInstance( - contextId, eu.dnetlib.dhp.oa.model.graph.Constants.CONTEXT_ENTITY), - Node.newInstance(ds, nodeType), + + contextId, eu.dnetlib.dhp.oa.model.graph.Constants.CONTEXT_ENTITY, + ds, nodeType, RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), Provenance .newInstance( @@ -77,10 +76,8 @@ public class Process implements Serializable { .add( Relation .newInstance( - Node.newInstance(ds, nodeType), - Node - .newInstance( - contextId, eu.dnetlib.dhp.oa.model.graph.Constants.CONTEXT_ENTITY), + ds, nodeType, + contextId, eu.dnetlib.dhp.oa.model.graph.Constants.CONTEXT_ENTITY, RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), Provenance .newInstance( diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java index ade2fb9..f6f7c9f 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java @@ -116,7 +116,7 @@ public class SparkCollectAndSave implements Serializable { .union(Utils.readPath(spark, inputPath + "/relation/context", Relation.class)) .union(Utils.readPath(spark, inputPath + "/relation/relation", Relation.class)); - Utils.getValidRelations(spark, relations, Utils.getEntitiesId(spark, outputPath)) + Utils.getValidRelations(relations, Utils.getEntitiesId(spark, outputPath)) // Dataset relJoinSource = relations // .joinWith(dumpedIds, relations.col("source.id").equalTo(dumpedIds.col("value"))) // .map((MapFunction, Relation>) t2 -> t2._1(), diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java index ab5eb7c..6f9a0e0 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java @@ -396,7 +396,7 @@ public class SparkDumpEntitiesJob implements Serializable { Optional .ofNullable(p.getId()) - .ifPresent(id -> project.setId(id)); + .ifPresent(id -> project.setId(id.substring(3))); Optional .ofNullable(p.getWebsiteurl()) @@ -502,6 +502,9 @@ public class SparkDumpEntitiesJob implements Serializable { } project.setFunding(funList); + if (Optional.ofNullable(p.getMeasures()).isPresent()) { + + } return project; } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java index c39f7c5..5a2c073 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java @@ -84,18 +84,12 @@ public class SparkDumpRelationJob implements Serializable { .map((MapFunction) relation -> { eu.dnetlib.dhp.oa.model.graph.Relation relNew = new eu.dnetlib.dhp.oa.model.graph.Relation(); relNew - .setSource( - Node - .newInstance( - relation.getSource(), - ModelSupport.idPrefixEntity.get(relation.getSource().substring(0, 2)))); + .setSource(relation.getSource().substring(3)); + relNew.setSourceType(ModelSupport.idPrefixEntity.get(relation.getSource().substring(0, 2))); relNew - .setTarget( - Node - .newInstance( - relation.getTarget(), - ModelSupport.idPrefixEntity.get(relation.getTarget().substring(0, 2)))); + .setTarget(relation.getTarget().substring(3)); + relNew.setTargetType(ModelSupport.idPrefixEntity.get(relation.getTarget().substring(0, 2))); relNew .setReltype( diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java index d0ae79d..7a0750f 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java @@ -155,8 +155,9 @@ public class SparkOrganizationRelation implements Serializable { .add( eu.dnetlib.dhp.oa.model.graph.Relation .newInstance( - Node.newInstance(id, Constants.CONTEXT_ENTITY), - Node.newInstance(organization, ModelSupport.idPrefixEntity.get(organization.substring(0, 2))), + id, Constants.CONTEXT_ENTITY, + organization, + ModelSupport.idPrefixEntity.get(organization.substring(0, 2)), RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), Provenance .newInstance( @@ -167,8 +168,8 @@ public class SparkOrganizationRelation implements Serializable { .add( eu.dnetlib.dhp.oa.model.graph.Relation .newInstance( - Node.newInstance(organization, ModelSupport.idPrefixEntity.get(organization.substring(0, 2))), - Node.newInstance(id, Constants.CONTEXT_ENTITY), + organization, ModelSupport.idPrefixEntity.get(organization.substring(0, 2)), + id, Constants.CONTEXT_ENTITY, RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), Provenance .newInstance( diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelation.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelation.java index 8e1db7b..022d6d6 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelation.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelation.java @@ -73,7 +73,7 @@ public class SparkSelectValidRelation implements Serializable { // read the results getValidRelations( - spark, Utils + Utils .readPath(spark, relationPath, Relation.class), getEntitiesId(spark, inputPath)) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelationContext.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelationContext.java index 073782d..779c4c2 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelationContext.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelationContext.java @@ -108,7 +108,7 @@ public class SparkSelectValidRelationContext implements Serializable { .readPath(spark, contextRelationPath + "/context", Relation.class) .union(Utils.readPath(spark, contextRelationPath + "/contextOrg", Relation.class)) .map( - (MapFunction>) r -> new Tuple2<>(r.getSource().getId(), r), + (MapFunction>) r -> new Tuple2<>(r.getSource(), r), Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))); Dataset allowedContext = Utils @@ -118,7 +118,7 @@ public class SparkSelectValidRelationContext implements Serializable { .joinWith(dumpedIds, relationSource.col("_1").equalTo(dumpedIds.col("value"))) .map( (MapFunction, String>, Tuple2>) t2 -> new Tuple2<>( - t2._1()._2().getTarget().getId(), t2._1()._2()), + t2._1()._2().getTarget(), t2._1()._2()), Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))); relJoinSource @@ -135,7 +135,7 @@ public class SparkSelectValidRelationContext implements Serializable { .joinWith(allowedContext, relationSource.col("_1").equalTo(allowedContext.col("id"))) .map( (MapFunction, ResearchCommunity>, Tuple2>) t2 -> new Tuple2<>( - t2._1()._2().getTarget().getId(), t2._1()._2()), + t2._1()._2().getTarget(), t2._1()._2()), Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))); relJoinSource diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java index a47c572..9f5c05b 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java @@ -356,7 +356,7 @@ public class DumpJobTest { Assertions.assertTrue(null == gr.getGeolocation() || gr.getGeolocation().size() == 0); - Assertions.assertEquals("50|pensoft_____::00ea4a1cd53806a97d62ea6bf268f2a2", gr.getId()); + Assertions.assertEquals("pensoft_____::00ea4a1cd53806a97d62ea6bf268f2a2", gr.getId()); Assertions.assertEquals(1, gr.getOriginalId().size()); Assertions @@ -1028,9 +1028,9 @@ public class DumpJobTest { Assertions.assertTrue(temp.count() == 2); - Assertions.assertTrue(temp.filter("id = '50|datacite____::05c611fdfc93d7a2a703d1324e28104a'").count() == 1); + Assertions.assertTrue(temp.filter("id = 'datacite____::05c611fdfc93d7a2a703d1324e28104a'").count() == 1); - Assertions.assertTrue(temp.filter("id = '50|dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'").count() == 1); + Assertions.assertTrue(temp.filter("id = 'dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'").count() == 1); temp = spark .sql( @@ -1043,7 +1043,7 @@ public class DumpJobTest { .assertEquals( "3131.64", temp - .filter("id = '50|datacite____::05c611fdfc93d7a2a703d1324e28104a'") + .filter("id = 'datacite____::05c611fdfc93d7a2a703d1324e28104a'") .collectAsList() .get(0) .getString(1)); @@ -1051,7 +1051,7 @@ public class DumpJobTest { .assertEquals( "EUR", temp - .filter("id = '50|datacite____::05c611fdfc93d7a2a703d1324e28104a'") + .filter("id = 'datacite____::05c611fdfc93d7a2a703d1324e28104a'") .collectAsList() .get(0) .getString(2)); @@ -1060,7 +1060,7 @@ public class DumpJobTest { .assertEquals( "2578.35", temp - .filter("id = '50|dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'") + .filter("id = 'dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'") .collectAsList() .get(0) .getString(1)); @@ -1068,7 +1068,7 @@ public class DumpJobTest { .assertEquals( "EUR", temp - .filter("id = '50|dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'") + .filter("id = 'dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'") .collectAsList() .get(0) .getString(2)); diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateRelationTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateRelationTest.java index c8aa41b..e01cb3e 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateRelationTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateRelationTest.java @@ -562,7 +562,7 @@ class CreateRelationTest { .assertTrue( rList .stream() - .map(r -> r.getSource().getId()) + .map(r -> r.getSource()) .collect(Collectors.toSet()) .contains( String @@ -579,7 +579,7 @@ class CreateRelationTest { .filter( r -> r .getSource() - .getId() + .equals( String .format( @@ -597,7 +597,7 @@ class CreateRelationTest { .filter( r -> r .getTarget() - .getId() + .equals( String .format( @@ -612,14 +612,14 @@ class CreateRelationTest { .filter( r -> r .getSource() - .getId() + .equals( String .format( "%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("dh-ch")))) - .map(r -> r.getTarget().getId()) + .map(r -> r.getTarget()) .collect(Collectors.toSet()); Assertions @@ -657,7 +657,7 @@ class CreateRelationTest { .assertFalse( rList .stream() - .map(r -> r.getSource().getId()) + .map(r -> r.getSource()) .collect(Collectors.toSet()) .contains( String @@ -674,7 +674,7 @@ class CreateRelationTest { .filter( r -> r .getSource() - .getId() + .equals( String .format( @@ -692,7 +692,7 @@ class CreateRelationTest { .filter( r -> r .getTarget() - .getId() + .equals( String .format( @@ -707,14 +707,14 @@ class CreateRelationTest { .filter( r -> r .getSource() - .getId() + .equals( String .format( "%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("clarin")))) - .map(r -> r.getTarget().getId()) + .map(r -> r.getTarget()) .collect(Collectors.toSet()); Assertions @@ -723,8 +723,8 @@ class CreateRelationTest { tmp.contains("40|corda_______::ef782b2d85676aa3e5a907427feb18c4")); rList.forEach(rel -> { - if (rel.getSource().getId().startsWith("40|")) { - String proj = rel.getSource().getId().substring(3); + if (rel.getSource().startsWith("40|")) { + String proj = rel.getSource().substring(3); Assertions.assertTrue(proj.substring(0, proj.indexOf("::")).length() == 12); } }); diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpRelationTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpRelationTest.java index a768ab1..2ac7afd 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpRelationTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpRelationTest.java @@ -97,7 +97,7 @@ public class DumpRelationTest { Dataset check = spark .sql( - "SELECT reltype.name, source.id source, source.type stype, target.id target,target.type ttype, provenance.provenance " + "SELECT reltype.name, source, sourceType stype, target,targetType ttype, provenance.provenance " + "from table "); @@ -158,7 +158,7 @@ public class DumpRelationTest { Dataset check = spark .sql( - "SELECT reltype.name, source.id source, source.type stype, target.id target,target.type ttype, provenance.provenance " + "SELECT reltype.name, source, sourceType stype, target,targetType ttype, provenance.provenance " + "from table "); @@ -229,7 +229,7 @@ public class DumpRelationTest { Dataset check = spark .sql( - "SELECT reltype.name, source.id source, source.type stype, target.id target,target.type ttype, provenance.provenance " + "SELECT reltype.name, source, sourceType stype, target,targetType ttype, provenance.provenance " + "from table "); @@ -283,7 +283,7 @@ public class DumpRelationTest { Dataset check = spark .sql( - "SELECT reltype.name, source.id source, source.type stype, target.id target,target.type ttype, provenance.provenance " + "SELECT reltype.name, source, sourceType stype, target,targetType ttype, provenance.provenance " + "from table "); diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/ExtractRelationFromEntityTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/ExtractRelationFromEntityTest.java index 49fae57..01a9f5d 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/ExtractRelationFromEntityTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/ExtractRelationFromEntityTest.java @@ -90,20 +90,22 @@ public class ExtractRelationFromEntityTest { org.apache.spark.sql.Dataset verificationDataset = spark .createDataset(tmp.rdd(), Encoders.bean(Relation.class)); - Assertions - .assertEquals( - 9, - verificationDataset.filter("source.id = '50|dedup_wf_001::15270b996fa8fd2fb5723daeab3685c3'").count()); + verificationDataset.show(false); Assertions .assertEquals( 9, - verificationDataset.filter("source.id = '50|dedup_wf_001::15270b996fa8fd2fb5723daxab3685c3'").count()); + verificationDataset.filter("source = 'dedup_wf_001::15270b996fa8fd2fb5723daeab3685c3'").count()); + + Assertions + .assertEquals( + 9, + verificationDataset.filter("source = 'dedup_wf_001::15270b996fa8fd2fb5723daxab3685c3'").count()); Assertions .assertEquals( "IsRelatedTo", verificationDataset - .filter((FilterFunction) row -> row.getSource().getId().startsWith("00")) + .filter((FilterFunction) row -> row.getSourceType().equals("context")) .collectAsList() .get(0) .getReltype() @@ -112,7 +114,7 @@ public class ExtractRelationFromEntityTest { Assertions .assertEquals( "relationship", verificationDataset - .filter((FilterFunction) row -> row.getSource().getId().startsWith("00")) + .filter((FilterFunction) row -> row.getSourceType().equals("context")) .collectAsList() .get(0) .getReltype() @@ -121,24 +123,22 @@ public class ExtractRelationFromEntityTest { Assertions .assertEquals( "context", verificationDataset - .filter((FilterFunction) row -> row.getSource().getId().startsWith("00")) + .filter((FilterFunction) row -> row.getSourceType().equals("context")) .collectAsList() .get(0) - .getSource() - .getType()); + .getSourceType()); Assertions .assertEquals( "result", verificationDataset - .filter((FilterFunction) row -> row.getSource().getId().startsWith("00")) + .filter((FilterFunction) row -> row.getSourceType().equals("context")) .collectAsList() .get(0) - .getTarget() - .getType()); + .getTargetType()); Assertions .assertEquals( "IsRelatedTo", verificationDataset - .filter((FilterFunction) row -> row.getTarget().getId().startsWith("00")) + .filter((FilterFunction) row -> row.getTargetType().equals("context")) .collectAsList() .get(0) .getReltype() @@ -147,7 +147,7 @@ public class ExtractRelationFromEntityTest { Assertions .assertEquals( "relationship", verificationDataset - .filter((FilterFunction) row -> row.getTarget().getId().startsWith("00")) + .filter((FilterFunction) row -> row.getTargetType().equals("context")) .collectAsList() .get(0) .getReltype() @@ -156,20 +156,18 @@ public class ExtractRelationFromEntityTest { Assertions .assertEquals( "context", verificationDataset - .filter((FilterFunction) row -> row.getTarget().getId().startsWith("00")) + .filter((FilterFunction) row -> row.getTargetType().equals("context")) .collectAsList() .get(0) - .getTarget() - .getType()); + .getTargetType()); Assertions .assertEquals( "result", verificationDataset - .filter((FilterFunction) row -> row.getTarget().getId().startsWith("00")) + .filter((FilterFunction) row -> row.getTargetType().equals("context")) .collectAsList() .get(0) - .getSource() - .getType()); + .getSourceType()); } @Test diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/subset/DumpSubsetTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/subset/DumpSubsetTest.java index 0b3c479..a9dcb5e 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/subset/DumpSubsetTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/subset/DumpSubsetTest.java @@ -472,19 +472,20 @@ public class DumpSubsetTest { .textFile(workingDir.toString() + "/dump/relation") .map(item -> OBJECT_MAPPER.readValue(item, eu.dnetlib.dhp.oa.model.graph.Relation.class)); + Assertions.assertEquals(10, tmp.count()); - Assertions.assertEquals(5, tmp.filter(r -> r.getSource().getId().startsWith("00")).count()); - Assertions.assertEquals(5, tmp.filter(r -> r.getTarget().getId().startsWith("00")).count()); + Assertions.assertEquals(5, tmp.filter(r -> r.getSourceType().equals("context")).count()); + Assertions.assertEquals(5, tmp.filter(r -> r.getTargetType().equals("context")).count()); - Assertions.assertEquals(2, tmp.filter(r -> r.getSource().getId().startsWith("10")).count()); - Assertions.assertEquals(2, tmp.filter(r -> r.getTarget().getId().startsWith("10")).count()); + Assertions.assertEquals(2, tmp.filter(r -> r.getSourceType().equals("datasource")).count()); + Assertions.assertEquals(2, tmp.filter(r -> r.getTargetType().equals("datasource")).count()); - Assertions.assertEquals(1, tmp.filter(r -> r.getSource().getId().startsWith("40")).count()); - Assertions.assertEquals(1, tmp.filter(r -> r.getTarget().getId().startsWith("40")).count()); + Assertions.assertEquals(1, tmp.filter(r -> r.getSourceType().equals("project")).count()); + Assertions.assertEquals(1, tmp.filter(r -> r.getTargetType().equals("project")).count()); - Assertions.assertEquals(2, tmp.filter(r -> r.getSource().getId().startsWith("20")).count()); - Assertions.assertEquals(2, tmp.filter(r -> r.getTarget().getId().startsWith("20")).count()); + Assertions.assertEquals(2, tmp.filter(r -> r.getSourceType().equals("organization")).count()); + Assertions.assertEquals(2, tmp.filter(r -> r.getTargetType().equals("organization")).count()); } @@ -514,9 +515,9 @@ public class DumpSubsetTest { Assertions.assertEquals(102, tmp.count()); - Assertions.assertEquals(51, tmp.filter(r -> r.getSource().getId().startsWith("50|")).count()); - Assertions.assertEquals(39, tmp.filter(r -> r.getSource().getId().startsWith("10|")).count()); - Assertions.assertEquals(12, tmp.filter(r -> r.getSource().getId().startsWith("00|")).count()); + Assertions.assertEquals(51, tmp.filter(r -> r.getSourceType().equals("result") ).count()); + Assertions.assertEquals(39, tmp.filter(r -> r.getSourceType().equals("datasource")).count()); + Assertions.assertEquals(12, tmp.filter(r -> r.getSourceType().equals("context")).count()); } } diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/community_infrastructure b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/community_infrastructure index d72fa66..cc7566b 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/community_infrastructure +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/community_infrastructure @@ -1,6 +1,6 @@ -{"id":"00|context_____::e15922110564cf669aaed346e871bc01","acronym":"eutopia","name":"EUTOPIA Open Research Portal","type":"Research Community","description":"

EUTOPIA is an ambitious alliance of 10 like-minded universities ready to reinvent themselves: the Babeș-Bolyai University in Cluj-Napoca (Romania), the Vrije Universiteit Brussels (Belgium), the Ca'Foscari University of Europe (Italy), CY Cergy Paris Université (France), the Technische Universität Dresden (Germany), the University of Gothenburg (Sweden), the University of Ljubljana (Slovenia), the NOVA University Lisbon (Portugal), the University of Pompeu Fabra (Spain) and the University of Warwick (United Kingdom). Together, these 10 pioneers join forces to build the university of the future.

","zenodo_community":null} -{"id":"00|context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","acronym":"enermaps","name":"Welcome to EnerMaps Gateway! Find the latest scientific data.","type":"Research Community","description":"","zenodo_community":null,"subject":[]} -{"id":"00|context_____::6f567d9abd1c6603b0c0205a832bc757","acronym":"neanias-underwater","name":"NEANIAS Underwater Research Community","type":"Research Community","description":"","zenodo_community":null,"subject":["Ocean mapping","Multibeam Backscatter","Bathymetry","Seabed classification","Submarine Geomorphology","Underwater Photogrammetry"]} -{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","acronym":"dh-ch","name":"Digital Humanities and Cultural Heritage","type":"Research Community","description":"This community gathers research results, data, scientific publications and projects related to the domain of Digital Humanities. This broad definition includes Humanities, Cultural Heritage, History, Archaeology and related fields.","zenodo_community":"https://zenodo.org/communities/oac_dh-ch","subject":["modern art","monuments","europeana data model","field walking","frescoes","LIDO metadata schema","art history","excavation","Arts and Humanities General","coins","temples","numismatics","lithics","environmental archaeology","digital cultural heritage","archaeological reports","history","CRMba","churches","cultural heritage","archaeological stratigraphy","religious art","digital humanities","archaeological sites","linguistic studies","bioarchaeology","architectural orders","palaeoanthropology","fine arts","europeana","CIDOC CRM","decorations","classic art","stratigraphy","digital archaeology","intangible cultural heritage","walls","chapels","CRMtex","Language and Literature","paintings","archaeology","mosaics","burials","medieval art","castles","CARARE metadata schema","statues","natural language processing","inscriptions","CRMsci","vaults","contemporary art","Arts and Humanities","CRMarchaeo","pottery"]} -{"id":"00|context_____::5fde864866ea5ded4cc873b3170b63c3","acronym":"beopen","name":"Transport Research","type":"Research Community","description":"Welcome to the Open Research Gateway for Transport Research. This gateway is part of the TOPOS Observatory (https://www.topos-observatory.eu). The TOPOS aims to showcase the status and progress of open science uptake in transport research. It focuses on promoting territorial and cross border cooperation and contributing in the optimization of open science in transport research.\nThe TOPOS Observatory is supported by the EC H2020 BEOPEN project (824323)","zenodo_community":"https://zenodo.org/communities/be-open-transport","subject":["Green Transport","City mobility systems","Vulnerable road users","Traffic engineering","Transport electrification","Intermodal freight transport","Clean vehicle fleets","Intelligent mobility","Inflight refueling","District mobility systems","Navigation and control systems for optimised planning and routing","European Space Technology Platform","European Transport networks","Green cars","Inter-modality infrastructures","Advanced Take Off and Landing Ideas","Sustainable urban systems","port-area railway networks","Innovative forms of urban transport","Alliance for Logistics Innovation through Collaboration in Europe","Advisory Council for Aeronautics Research in Europe","Mobility services for people and goods","Guidance and traffic management","Passenger mobility","Smart mobility and services","transport innovation","high-speed railway","Vehicle design","Inland shipping","public transportation","aviation’s climate impact","Road transport","On-demand public transport","Personal Air Transport","Pipeline transport","European Association of Aviation Training and Education Organisations","Defrosting of railway infrastructure","Inclusive and affordable transport","River Information Services","jel:L92","Increased use of public transport","Seamless mobility","STRIA","trolleybus transport","Intelligent Transport System","Low-emission alternative energy for transport","Shared mobility for people and goods","Business model for urban mobility","Interoperability of transport systems","Cross-border train slot booking","Air transport","Transport pricing","Sustainable transport","European Rail Transport Research Advisory Council","Alternative aircraft configurations","Railways applications","urban transport","Environmental impact of transport","urban freight delivery systems","Automated Road Transport","Alternative fuels in public transport","Active LIDAR-sensor for GHG-measurements","Autonomous logistics operations","Rational use of motorised transport","Network and traffic management systems","electrification of railway wagons","Single European Sky","Electrified road systems","Railway dynamics","Motorway of the Sea","smart railway communications","Maritime transport","Environmental- friendly transport","Combined transport","Connected automated driving technology","Innovative freight logistics services","automated and shared vehicles","Alternative Aircraft Systems","Land-use and transport interaction","Public transport system","Business plan for shared mobility","Shared mobility","Growing of mobility demand","European Road Transport Research Advisory Council","WATERBORNE ETP","Effective transport management system","Short Sea Shipping","air traffic management","Sea hubs and the motorways of the sea","Urban mobility solutions","Smart city planning","Maritime spatial planning","EUropean rail Research Network of Excellence","ENERGY CONSUMPTION BY THE TRANSPORT SECTOR","Integrated urban plan","inland waterway services","European Conference of Transport Research Institutes","air vehicles","E-freight","Automated Driving","Automated ships","pricing for cross-border passenger transport","Vehicle efficiency","Railway transport","Electric vehicles","Road traffic monitoring","Deep sea shipping","Circular economy in transport","Traffic congestion","air transport system","Urban logistics","Rail transport","OpenStreetMap","high speed rail","Transportation engineering","Intermodal travel information","Flight Data Recorders","Advanced driver assistance systems","long distance freight transport","Inland waterway transport","Smart mobility","Mobility integration","Personal Rapid Transit system","Safety measures & requirements for roads","Green rail transport","Vehicle manufacturing","Future Airport Layout","Rail technologies","European Intermodal Research Advisory Council","inland navigation","Automated urban vehicles","ECSS-standards","Traveller services","Polluting transport","Air Traffic Control","Cooperative and connected and automated transport","Innovative powertrains","Quality of transport system and services","door-to- door logistics chain","Inter-modal aspects of urban mobility","Innovative freight delivery systems","urban freight delivery infrastructures"]} -{"id":"00|context_____::a38bf77184799906a6ce86b9eb761c80","acronym":"sdsn-gr","name":"Sustainable Development Solutions Network - Greece","type":"Research Community","description":"The UN Sustainable Development Solutions Network (SDSN) has been operating since 2012 under the auspices of the UN Secretary-General. SDSN mobilizes global scientific and technological expertise to promote practical solutions for sustainable development, including the implementation of the Sustainable Development Goals (SDGs) and the Paris Climate Agreement. The Greek hub of SDSN has been included in the SDSN network in 2017 and is co-hosted by ICRE8: International Center for Research on the Environment and the Economy and the Political Economy of Sustainable Development Lab.","zenodo_community":"https://zenodo.org/communities/oac_sdsn-greece","subject":["SDG13 - Climate action","SDG8 - Decent work and economic\n\t\t\t\t\tgrowth","SDG15 - Life on land","SDG2 - Zero hunger","SDG17 - Partnerships for the\n\t\t\t\t\tgoals","SDG10 - Reduced inequalities","SDG5 - Gender equality","SDG12 - Responsible\n\t\t\t\t\tconsumption and production","SDG14 - Life below water","SDG6 - Clean water and\n\t\t\t\t\tsanitation","SDG11 - Sustainable cities and communities","SDG1 - No poverty","SDG3 -\n\t\t\t\t\tGood health and well being","SDG7 - Affordable and clean energy","SDG4 - Quality\n\t\t\t\t\teducation","SDG9 - Industry innovation and infrastructure","SDG16 - Peace justice\n\t\t\t\t\tand strong institutions"]} \ No newline at end of file +{"id":"context_____::e15922110564cf669aaed346e871bc01","acronym":"eutopia","name":"EUTOPIA Open Research Portal","type":"Research Community","description":"

EUTOPIA is an ambitious alliance of 10 like-minded universities ready to reinvent themselves: the Babeș-Bolyai University in Cluj-Napoca (Romania), the Vrije Universiteit Brussels (Belgium), the Ca'Foscari University of Europe (Italy), CY Cergy Paris Université (France), the Technische Universität Dresden (Germany), the University of Gothenburg (Sweden), the University of Ljubljana (Slovenia), the NOVA University Lisbon (Portugal), the University of Pompeu Fabra (Spain) and the University of Warwick (United Kingdom). Together, these 10 pioneers join forces to build the university of the future.

","zenodo_community":null} +{"id":"context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","acronym":"enermaps","name":"Welcome to EnerMaps Gateway! Find the latest scientific data.","type":"Research Community","description":"","zenodo_community":null,"subject":[]} +{"id":"context_____::6f567d9abd1c6603b0c0205a832bc757","acronym":"neanias-underwater","name":"NEANIAS Underwater Research Community","type":"Research Community","description":"","zenodo_community":null,"subject":["Ocean mapping","Multibeam Backscatter","Bathymetry","Seabed classification","Submarine Geomorphology","Underwater Photogrammetry"]} +{"id":"context_____::04a00617ca659adc944977ac700ea14b","acronym":"dh-ch","name":"Digital Humanities and Cultural Heritage","type":"Research Community","description":"This community gathers research results, data, scientific publications and projects related to the domain of Digital Humanities. This broad definition includes Humanities, Cultural Heritage, History, Archaeology and related fields.","zenodo_community":"https://zenodo.org/communities/oac_dh-ch","subject":["modern art","monuments","europeana data model","field walking","frescoes","LIDO metadata schema","art history","excavation","Arts and Humanities General","coins","temples","numismatics","lithics","environmental archaeology","digital cultural heritage","archaeological reports","history","CRMba","churches","cultural heritage","archaeological stratigraphy","religious art","digital humanities","archaeological sites","linguistic studies","bioarchaeology","architectural orders","palaeoanthropology","fine arts","europeana","CIDOC CRM","decorations","classic art","stratigraphy","digital archaeology","intangible cultural heritage","walls","chapels","CRMtex","Language and Literature","paintings","archaeology","mosaics","burials","medieval art","castles","CARARE metadata schema","statues","natural language processing","inscriptions","CRMsci","vaults","contemporary art","Arts and Humanities","CRMarchaeo","pottery"]} +{"id":"context_____::5fde864866ea5ded4cc873b3170b63c3","acronym":"beopen","name":"Transport Research","type":"Research Community","description":"Welcome to the Open Research Gateway for Transport Research. This gateway is part of the TOPOS Observatory (https://www.topos-observatory.eu). The TOPOS aims to showcase the status and progress of open science uptake in transport research. It focuses on promoting territorial and cross border cooperation and contributing in the optimization of open science in transport research.\nThe TOPOS Observatory is supported by the EC H2020 BEOPEN project (824323)","zenodo_community":"https://zenodo.org/communities/be-open-transport","subject":["Green Transport","City mobility systems","Vulnerable road users","Traffic engineering","Transport electrification","Intermodal freight transport","Clean vehicle fleets","Intelligent mobility","Inflight refueling","District mobility systems","Navigation and control systems for optimised planning and routing","European Space Technology Platform","European Transport networks","Green cars","Inter-modality infrastructures","Advanced Take Off and Landing Ideas","Sustainable urban systems","port-area railway networks","Innovative forms of urban transport","Alliance for Logistics Innovation through Collaboration in Europe","Advisory Council for Aeronautics Research in Europe","Mobility services for people and goods","Guidance and traffic management","Passenger mobility","Smart mobility and services","transport innovation","high-speed railway","Vehicle design","Inland shipping","public transportation","aviation’s climate impact","Road transport","On-demand public transport","Personal Air Transport","Pipeline transport","European Association of Aviation Training and Education Organisations","Defrosting of railway infrastructure","Inclusive and affordable transport","River Information Services","jel:L92","Increased use of public transport","Seamless mobility","STRIA","trolleybus transport","Intelligent Transport System","Low-emission alternative energy for transport","Shared mobility for people and goods","Business model for urban mobility","Interoperability of transport systems","Cross-border train slot booking","Air transport","Transport pricing","Sustainable transport","European Rail Transport Research Advisory Council","Alternative aircraft configurations","Railways applications","urban transport","Environmental impact of transport","urban freight delivery systems","Automated Road Transport","Alternative fuels in public transport","Active LIDAR-sensor for GHG-measurements","Autonomous logistics operations","Rational use of motorised transport","Network and traffic management systems","electrification of railway wagons","Single European Sky","Electrified road systems","Railway dynamics","Motorway of the Sea","smart railway communications","Maritime transport","Environmental- friendly transport","Combined transport","Connected automated driving technology","Innovative freight logistics services","automated and shared vehicles","Alternative Aircraft Systems","Land-use and transport interaction","Public transport system","Business plan for shared mobility","Shared mobility","Growing of mobility demand","European Road Transport Research Advisory Council","WATERBORNE ETP","Effective transport management system","Short Sea Shipping","air traffic management","Sea hubs and the motorways of the sea","Urban mobility solutions","Smart city planning","Maritime spatial planning","EUropean rail Research Network of Excellence","ENERGY CONSUMPTION BY THE TRANSPORT SECTOR","Integrated urban plan","inland waterway services","European Conference of Transport Research Institutes","air vehicles","E-freight","Automated Driving","Automated ships","pricing for cross-border passenger transport","Vehicle efficiency","Railway transport","Electric vehicles","Road traffic monitoring","Deep sea shipping","Circular economy in transport","Traffic congestion","air transport system","Urban logistics","Rail transport","OpenStreetMap","high speed rail","Transportation engineering","Intermodal travel information","Flight Data Recorders","Advanced driver assistance systems","long distance freight transport","Inland waterway transport","Smart mobility","Mobility integration","Personal Rapid Transit system","Safety measures & requirements for roads","Green rail transport","Vehicle manufacturing","Future Airport Layout","Rail technologies","European Intermodal Research Advisory Council","inland navigation","Automated urban vehicles","ECSS-standards","Traveller services","Polluting transport","Air Traffic Control","Cooperative and connected and automated transport","Innovative powertrains","Quality of transport system and services","door-to- door logistics chain","Inter-modal aspects of urban mobility","Innovative freight delivery systems","urban freight delivery infrastructures"]} +{"id":"context_____::a38bf77184799906a6ce86b9eb761c80","acronym":"sdsn-gr","name":"Sustainable Development Solutions Network - Greece","type":"Research Community","description":"The UN Sustainable Development Solutions Network (SDSN) has been operating since 2012 under the auspices of the UN Secretary-General. SDSN mobilizes global scientific and technological expertise to promote practical solutions for sustainable development, including the implementation of the Sustainable Development Goals (SDGs) and the Paris Climate Agreement. The Greek hub of SDSN has been included in the SDSN network in 2017 and is co-hosted by ICRE8: International Center for Research on the Environment and the Economy and the Political Economy of Sustainable Development Lab.","zenodo_community":"https://zenodo.org/communities/oac_sdsn-greece","subject":["SDG13 - Climate action","SDG8 - Decent work and economic\n\t\t\t\t\tgrowth","SDG15 - Life on land","SDG2 - Zero hunger","SDG17 - Partnerships for the\n\t\t\t\t\tgoals","SDG10 - Reduced inequalities","SDG5 - Gender equality","SDG12 - Responsible\n\t\t\t\t\tconsumption and production","SDG14 - Life below water","SDG6 - Clean water and\n\t\t\t\t\tsanitation","SDG11 - Sustainable cities and communities","SDG1 - No poverty","SDG3 -\n\t\t\t\t\tGood health and well being","SDG7 - Affordable and clean energy","SDG4 - Quality\n\t\t\t\t\teducation","SDG9 - Industry innovation and infrastructure","SDG16 - Peace justice\n\t\t\t\t\tand strong institutions"]} \ No newline at end of file diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/datasource b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/datasource index 5f5a354..3c1f311 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/datasource +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/datasource @@ -1,4 +1,4 @@ -{"id":"10|doajarticles::9c4b678901e5276d9e3addee566816af","originalId":["doajarticles::1798-355X"],"pid":[],"datasourcetype":{"scheme":"pubsrepository::journal","value":"Journal"},"openairecompatibility":"not available","officialname":"Pelitutkimuksen vuosikirja","englishname":"Pelitutkimuksen vuosikirja","websiteurl":"http://www.pelitutkimus.fi","logourl":null,"dateofvalidation":null,"description":null,"subjects":["Geography. Anthropology. Recreation: Recreation. Leisure | Science: Mathematics: Instruments and machines: Electronic computers. Computer science: Computer software"],"languages":[],"contenttypes":["Journal articles"],"releasestartdate":null,"releaseenddate":null,"missionstatementurl":null,"accessrights":null,"uploadrights":null,"databaseaccessrestriction":null,"datauploadrestriction":null,"versioning":false,"citationguidelineurl":null,"pidsystems":null,"certificates":null,"policies":[],"journal":null} -{"id":"10|doajarticles::acb7c79bb85d3b3a7b75389f5d9570f5","originalId":["doajarticles::1879-9337"],"pid":[],"datasourcetype":{"scheme":"pubsrepository::journal","value":"Journal"},"openairecompatibility":"collected from a compatible aggregator","officialname":"Review of Development Finance","englishname":"Review of Development Finance","websiteurl":"http://www.journals.elsevier.com/review-of-development-finance/","logourl":null,"dateofvalidation":null,"description":null,"subjects":["Social Sciences: Industries. Land use. Labor: Economic growth, development, planning | Social Sciences: Finance"],"languages":[],"contenttypes":["Journal articles"],"releasestartdate":null,"releaseenddate":null,"missionstatementurl":null,"accessrights":null,"uploadrights":null,"databaseaccessrestriction":null,"datauploadrestriction":null,"versioning":false,"citationguidelineurl":null,"pidsystems":null,"certificates":null,"policies":[],"journal":null} -{"id":"10|doajarticles::1fa6859d71faa77b32d82f278c6ed1df","originalId":["doajarticles::1048-9533"],"pid":[],"datasourcetype":{"scheme":"pubsrepository::journal","value":"Journal"},"openairecompatibility":"collected from a compatible aggregator","officialname":"Journal of Applied Mathematics and Stochastic Analysis","englishname":"Journal of Applied Mathematics and Stochastic Analysis","websiteurl":"https://www.hindawi.com/journals/jamsa","logourl":null,"dateofvalidation":null,"description":null,"subjects":[],"languages":[],"contenttypes":["Journal articles"],"releasestartdate":null,"releaseenddate":null,"missionstatementurl":null,"accessrights":null,"uploadrights":null,"databaseaccessrestriction":null,"datauploadrestriction":null,"versioning":false,"citationguidelineurl":null,"pidsystems":null,"certificates":null,"policies":[],"journal":null} -{"id":"10|doajarticles::a5314b60f79b869cb5d3a2709167bc3a","originalId":["doajarticles::0322-788X"],"pid":[],"datasourcetype":{"scheme":"pubsrepository::journal","value":"Journal"},"openairecompatibility":"collected from a compatible aggregator","officialname":"Statistika: Statistics and Economy Journal","englishname":"Statistika: Statistics and Economy Journal","websiteurl":"http://www.czso.cz/statistika_journal","logourl":null,"dateofvalidation":null,"description":null,"subjects":["Social Sciences: Statistics"],"languages":[],"contenttypes":["Journal articles"],"releasestartdate":null,"releaseenddate":null,"missionstatementurl":null,"accessrights":null,"uploadrights":null,"databaseaccessrestriction":null,"datauploadrestriction":null,"versioning":false,"citationguidelineurl":null,"pidsystems":null,"certificates":null,"policies":[],"journal":null} \ No newline at end of file +{"id":"doajarticles::9c4b678901e5276d9e3addee566816af","originalId":["doajarticles::1798-355X"],"pid":[],"datasourcetype":{"scheme":"pubsrepository::journal","value":"Journal"},"openairecompatibility":"not available","officialname":"Pelitutkimuksen vuosikirja","englishname":"Pelitutkimuksen vuosikirja","websiteurl":"http://www.pelitutkimus.fi","logourl":null,"dateofvalidation":null,"description":null,"subjects":["Geography. Anthropology. Recreation: Recreation. Leisure | Science: Mathematics: Instruments and machines: Electronic computers. Computer science: Computer software"],"languages":[],"contenttypes":["Journal articles"],"releasestartdate":null,"releaseenddate":null,"missionstatementurl":null,"accessrights":null,"uploadrights":null,"databaseaccessrestriction":null,"datauploadrestriction":null,"versioning":false,"citationguidelineurl":null,"pidsystems":null,"certificates":null,"policies":[],"journal":null} +{"id":"doajarticles::acb7c79bb85d3b3a7b75389f5d9570f5","originalId":["doajarticles::1879-9337"],"pid":[],"datasourcetype":{"scheme":"pubsrepository::journal","value":"Journal"},"openairecompatibility":"collected from a compatible aggregator","officialname":"Review of Development Finance","englishname":"Review of Development Finance","websiteurl":"http://www.journals.elsevier.com/review-of-development-finance/","logourl":null,"dateofvalidation":null,"description":null,"subjects":["Social Sciences: Industries. Land use. Labor: Economic growth, development, planning | Social Sciences: Finance"],"languages":[],"contenttypes":["Journal articles"],"releasestartdate":null,"releaseenddate":null,"missionstatementurl":null,"accessrights":null,"uploadrights":null,"databaseaccessrestriction":null,"datauploadrestriction":null,"versioning":false,"citationguidelineurl":null,"pidsystems":null,"certificates":null,"policies":[],"journal":null} +{"id":"doajarticles::1fa6859d71faa77b32d82f278c6ed1df","originalId":["doajarticles::1048-9533"],"pid":[],"datasourcetype":{"scheme":"pubsrepository::journal","value":"Journal"},"openairecompatibility":"collected from a compatible aggregator","officialname":"Journal of Applied Mathematics and Stochastic Analysis","englishname":"Journal of Applied Mathematics and Stochastic Analysis","websiteurl":"https://www.hindawi.com/journals/jamsa","logourl":null,"dateofvalidation":null,"description":null,"subjects":[],"languages":[],"contenttypes":["Journal articles"],"releasestartdate":null,"releaseenddate":null,"missionstatementurl":null,"accessrights":null,"uploadrights":null,"databaseaccessrestriction":null,"datauploadrestriction":null,"versioning":false,"citationguidelineurl":null,"pidsystems":null,"certificates":null,"policies":[],"journal":null} +{"id":"doajarticles::a5314b60f79b869cb5d3a2709167bc3a","originalId":["doajarticles::0322-788X"],"pid":[],"datasourcetype":{"scheme":"pubsrepository::journal","value":"Journal"},"openairecompatibility":"collected from a compatible aggregator","officialname":"Statistika: Statistics and Economy Journal","englishname":"Statistika: Statistics and Economy Journal","websiteurl":"http://www.czso.cz/statistika_journal","logourl":null,"dateofvalidation":null,"description":null,"subjects":["Social Sciences: Statistics"],"languages":[],"contenttypes":["Journal articles"],"releasestartdate":null,"releaseenddate":null,"missionstatementurl":null,"accessrights":null,"uploadrights":null,"databaseaccessrestriction":null,"datauploadrestriction":null,"versioning":false,"citationguidelineurl":null,"pidsystems":null,"certificates":null,"policies":[],"journal":null} \ No newline at end of file diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/organization b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/organization index da634c2..5e728c5 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/organization +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/organization @@ -1,3 +1,3 @@ -{"legalshortname":"SHoF","legalname":"Swedish House of Finance","websiteurl":"http://houseoffinance.se/","alternativenames":["SHoF"],"country":{"code":"SE","label":"Sweden"},"id":"20|grid________::87698402476531ba39e61f1df38f2a91","pid":[{"scheme":"grid","value":"grid.451954.8"}]} -{"legalshortname":"Korean Elementary Moral Education Society","legalname":"Korean Elementary Moral Education Society","websiteurl":"http://www.ethics.or.kr/","alternativenames":["한국초등도덕교육학회"],"country":{"code":"KR","label":"Korea (Republic of)"},"id":"20|grid________::bd5cbea5dc434b8fd811a880cb9d4a05","pid":[{"scheme":"grid","value":"grid.496778.3"}]} -{"legalshortname":"NHC","legalname":"National Health Council","websiteurl":"http://www.nationalhealthcouncil.org/","alternativenames":["NHC"],"country":{"code":"US","label":"United States"},"id":"20|grid________::94948cc036605bf4a00ec77ce5ca92d3","pid":[{"scheme":"grid","value":"grid.487707.b"}]} \ No newline at end of file +{"legalshortname":"SHoF","legalname":"Swedish House of Finance","websiteurl":"http://houseoffinance.se/","alternativenames":["SHoF"],"country":{"code":"SE","label":"Sweden"},"id":"grid________::87698402476531ba39e61f1df38f2a91","pid":[{"scheme":"grid","value":"grid.451954.8"}]} +{"legalshortname":"Korean Elementary Moral Education Society","legalname":"Korean Elementary Moral Education Society","websiteurl":"http://www.ethics.or.kr/","alternativenames":["한국초등도덕교육학회"],"country":{"code":"KR","label":"Korea (Republic of)"},"id":"grid________::bd5cbea5dc434b8fd811a880cb9d4a05","pid":[{"scheme":"grid","value":"grid.496778.3"}]} +{"legalshortname":"NHC","legalname":"National Health Council","websiteurl":"http://www.nationalhealthcouncil.org/","alternativenames":["NHC"],"country":{"code":"US","label":"United States"},"id":"grid________::94948cc036605bf4a00ec77ce5ca92d3","pid":[{"scheme":"grid","value":"grid.487707.b"}]} \ No newline at end of file diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/project b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/project index d09314b..549a83d 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/project +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/project @@ -1,3 +1,3 @@ -{"id":"40|aka_________::01bb7b48e29d732a1c7bc5150b9195c4","websiteurl":null,"code":"135027","acronym":null,"title":"Dynamic 3D resolution-enhanced low-coherence interferometric imaging / Consortium: Hi-Lo","startdate":null,"enddate":null,"callidentifier":"Fotoniikka ja modernit kuvantamismenetelmät LT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","funding_stream":null}],"summary":null,"granted":null,"h2020programme":[]} -{"id":"40|aka_________::9d1af21dbd0f5bc719f71553d19a6b3a","websiteurl":null,"code":"316061","acronym":null,"title":"Finnish Imaging of Degenerative Shoulder Study (FIMAGE): A study on the prevalence of degenerative imaging changes of the shoulder and their relevance to clinical symptoms in the general population.","startdate":null,"enddate":null,"callidentifier":"Academy Project Funding TT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","funding_stream":null}],"summary":null,"granted":null,"h2020programme":[]} -{"id":"40|anr_________::1f21edc5c902be305ee47148955c6e50","websiteurl":null,"code":"ANR-17-CE05-0033","acronym":"MOISE","title":"METAL OXIDES AS LOW LOADED NANO-IRIDIUM SUPPORT FOR COMPETITIVE WATER ELECTROLYSIS","startdate":null,"enddate":null,"callidentifier":null,"keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"ANR","name":"French National Research Agency (ANR)","jurisdiction":"FR","funding_stream":null}],"summary":null,"granted":null,"h2020programme":[]} \ No newline at end of file +{"id":"aka_________::01bb7b48e29d732a1c7bc5150b9195c4","websiteurl":null,"code":"135027","acronym":null,"title":"Dynamic 3D resolution-enhanced low-coherence interferometric imaging / Consortium: Hi-Lo","startdate":null,"enddate":null,"callidentifier":"Fotoniikka ja modernit kuvantamismenetelmät LT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","funding_stream":null}],"summary":null,"granted":null,"h2020programme":[]} +{"id":"aka_________::9d1af21dbd0f5bc719f71553d19a6b3a","websiteurl":null,"code":"316061","acronym":null,"title":"Finnish Imaging of Degenerative Shoulder Study (FIMAGE): A study on the prevalence of degenerative imaging changes of the shoulder and their relevance to clinical symptoms in the general population.","startdate":null,"enddate":null,"callidentifier":"Academy Project Funding TT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","funding_stream":null}],"summary":null,"granted":null,"h2020programme":[]} +{"id":"anr_________::1f21edc5c902be305ee47148955c6e50","websiteurl":null,"code":"ANR-17-CE05-0033","acronym":"MOISE","title":"METAL OXIDES AS LOW LOADED NANO-IRIDIUM SUPPORT FOR COMPETITIVE WATER ELECTROLYSIS","startdate":null,"enddate":null,"callidentifier":null,"keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"ANR","name":"French National Research Agency (ANR)","jurisdiction":"FR","funding_stream":null}],"summary":null,"granted":null,"h2020programme":[]} \ No newline at end of file diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/publication b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/publication index 0af68b8..aadfe07 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/publication +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/dump/publication @@ -1,15 +1,15 @@ -{"author":[{"fullname":"van Someren, Christian","name":"Christian","surname":"van Someren","rank":1,"pid":null}],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"energieproductie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Management"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Monitoring"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Policy and Law"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Energie interventies en gedrag"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"publieke ondersteuning en communicatie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Professional practice & society"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Energie opwek","description":["Over het Energieakkoord. In het energieakkoord voor duurzame groei is afgesproken dat in 2020 14 procent van de opwek hernieuwbaar moet zijn en in 2023 16 procent. De doelstelling is een uitdagende opgave waarbij de eerste vraag is: \"Hoeveel hernieuwbare energie wordt er op dit moment opgewekt in Nederland?\" Deze website geeft antwoord op de vraag voor de actueel opgewekte windenergie, zonne-energie en biogas."],"publicationdate":"2016-11-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"50|DansKnawCris::3c81248c335f0aa07e06817ece6fa6af","originalId":["DansKnawCris::3c81248c335f0aa07e06817ece6fa6af"],"pid":[{"scheme":"urn","value":"urn:nbn:nl:hs:18-813a5dfa-4fd0-44c4-8cbf-310324dc724d"},{"scheme":"urn","value":"urn:nbn:nl:hs:18-813a5dfa-4fd0-44c4-8cbf-310324dc724d"}],"dateofcollection":"","lastupdatetimestamp":1591282663379,"instance":[{"accessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Other literature type","url":["http://energieopwek.nl/"],"publicationdate":"2016-11-01"}]} -{"author":[],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Vlaardingen"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Plangebied Het Hof en Oranjepark : gemeente Vlaardingen : archeologisch vooronderzoek: een inventariserend veldonderzoek (verkennende fase)","description":["Met lit. opg"],"publicationdate":"2010-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"50|DansKnawCris::4669a378a73661417182c208e6fdab53","originalId":["DansKnawCris::4669a378a73661417182c208e6fdab53"],"pid":[{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceFullCatalogue&search=priref=800007467"},{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceFullCatalogue&search=priref=800007467"}],"dateofcollection":"","lastupdatetimestamp":1591282758835,"instance":[{"accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Report","publicationdate":"2010-01-01"}]} -{"author":[],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"prospectie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Honselersdijk tracé persleiding (gemeente Westland) : een bureauonderzoek en inventariserend veldonderzoek in de vorm van een verkennend en karterend booronderzoek","description":["Lit.opg."],"publicationdate":"2008-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"50|DansKnawCris::52c4541c9bffde34daa945ece8dcf635","originalId":["DansKnawCris::52c4541c9bffde34daa945ece8dcf635"],"pid":[{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550013915"},{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550013915"}],"dateofcollection":"","lastupdatetimestamp":1591283000091,"instance":[{"accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Report","publicationdate":"2008-01-01"}]} -{"author":[],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Archeologisch onderzoek plangebied Akker-Boekenderweg te Thorn","description":[],"publicationdate":"2012-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"50|DansKnawCris::794d07c2e66f1fbf07d61b9bfca36dc2","originalId":["DansKnawCris::794d07c2e66f1fbf07d61b9bfca36dc2"],"pid":[{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550028404"},{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550028404"}],"dateofcollection":"","lastupdatetimestamp":1591282847497,"instance":[{"accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Report","publicationdate":"2012-01-01"}]} -{"author":[{"fullname":"Zwieten, van, P.A.M.","name":"van, P.A.M.","surname":"Zwieten","rank":1,"pid":null},{"fullname":"Banda, M.","name":"M.","surname":"Banda","rank":2,"pid":null},{"fullname":"Kolding, J.","name":"J.","surname":"Kolding","rank":3,"pid":null}],"type":"publication","language":{"code":"en","label":"en"},"country":[],"subjects":[{"subject":{"scheme":"","value":"african great-lakes"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"cichlid fishes"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"reference points"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"east-africa"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"management"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"perspective"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"tanganyika"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"diversity"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"history"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"nyasa"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Selecting indicators to assess the fisheries of Lake Malawi and Lake Malombe: Knowledge base and evaluative capacity","description":["The provision of management information on the fisheries of Lakes Malawi and Malombe has been characterised by top–down controlled single species steady-state assessment techniques originating from single gear industrial fisheries but applied to an open access highly diverse and adaptive small-scale multispecies and multi-gear fishery. The result has largely been an unhappy marriage with uncertainties blamed more on the data than the process, although the data collection generally is detailed and comprehensive on catch and effort parameters. An extensive literature review of primary and grey literature on ecosystem drivers, exploitation pressures, and fish population and community states shows that Malawi has the necessary knowledge base for expanding their assessment into multi-causal and exploratory indicator-based methods that can assist in better understanding and more disciplined use of existing data and monitoring systems. Selection and ranking of a suite of indicators focusing on the major fisheries in the Southeast arm of Lake Malawi and Lake Malombe were done by a group of Malawian fisheries researchers and management advisers, thereby testing a framework of scoring criteria assessing an indicator's acceptability, observability, and relatedness to management. Indicators that are close to raw observational data and that require limited permutations and few assumptions appear to be preferable in the Malawian context. CPUE-based assessments can improve the utility of data and information in communicating developments and processes and evaluate fisheries management policies"],"publicationdate":"2011-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"container":{"name":"Journal of Great Lakes Research","issnPrinted":"0380-1330","issnOnline":"","issnLinking":"","ep":"44","iss":"1","sp":"26","vol":"37","edition":"","conferenceplace":null,"conferencedate":null},"id":"50|DansKnawCris::b90247718304c409331edb82fd0e8d56","originalId":["DansKnawCris::b90247718304c409331edb82fd0e8d56"],"pid":[{"scheme":"urn","value":"urn:nbn:nl:ui:32-401847"},{"scheme":"urn","value":"urn:nbn:nl:ui:32-401847"}],"dateofcollection":"","lastupdatetimestamp":1591282621858,"instance":[{"accessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Article","publicationdate":"2011-01-01"}]} -{"author":[],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Bureauonderzoek"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Clavecymbelstraat te Maastricht","description":[],"publicationdate":"2010-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"50|DansKnawCris::ba8f1ea3adf1bba501ec9da3a58e9638","originalId":["DansKnawCris::ba8f1ea3adf1bba501ec9da3a58e9638"],"pid":[{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550019564"},{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550019564"}],"dateofcollection":"","lastupdatetimestamp":1591282723014,"instance":[{"accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Report","publicationdate":"2010-01-01"}]} -{"author":[{"fullname":"Wegwarth, Odette","name":"Odette","surname":"Wegwarth","rank":1,"pid":null},{"fullname":"Pashayan, Nora","name":"Nora","surname":"Pashayan","rank":2,"pid":null},{"fullname":"Widschwendter, Martin","name":"Martin","surname":"Widschwendter","rank":3,"pid":null},{"fullname":"Rebitschek, Felix","name":"Felix","surname":"Rebitschek","rank":4,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Medicine"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Molecular Biology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Sociology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"19999 Mathematical Sciences not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Developmental Biology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Cancer"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Science Policy"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Additional file 1: of Womenâ s perception, attitudes, and intended behavior towards predictive epigenetic risk testing for female cancers in 5 European countries: a cross-sectional online survey","description":["STROBE checklist. (DOC 98 kb)"],"publicationdate":"2019-01-01","publisher":"Figshare","source":[],"format":[],"contributor":[],"coverage":[],"id":"50|datacite____::016dfd6ecbfaa929bbd90ec3a2b51521","originalId":["datacite____::016dfd6ecbfaa929bbd90ec3a2b51521"],"pid":[{"scheme":"doi","value":"10.6084/m9.figshare.8207303.v1"}],"dateofcollection":"","lastupdatetimestamp":1591282908770,"instance":[{"license":"https://creativecommons.org/licenses/by/4.0","type":"Other literature type","url":["http://dx.doi.org/10.6084/m9.figshare.8207303.v1"],"publicationdate":"2019-01-01"}]} -{"author":[{"fullname":"Archaeology South East","name":"","surname":"","rank":1,"pid":null}],"type":"publication","language":{"code":"en","label":"en"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Grey Literature"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"An Archaeological Evaluation at 290 -294 Golder's Green Road, London Borough of Barnet NW11","description":[],"publicationdate":"2007-01-01","publisher":"Archaeology Data Service","source":[],"format":["PDF"],"contributor":[],"coverage":[],"id":"50|datacite____::05c611fdfc93d7a2a703d1324e28104a","originalId":["datacite____::05c611fdfc93d7a2a703d1324e28104a"],"pid":[{"scheme":"doi","value":"10.5284/1003453"}],"dateofcollection":"","lastupdatetimestamp":1591282746270,"instance":[{"type":"Other literature type","url":["http://dx.doi.org/10.5284/1003453"],"articleprocessingcharge":{"currency":"EUR","amount":"3131.64"},"publicationdate":"2007-01-01"}]} -{"author":[{"fullname":"Veen, V. Van Der","name":"V.","surname":"Veen","rank":1,"pid":null}],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"verkennend booronderzoek"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Onbekend (XXX)"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Archeologisch bureau- en verkennend veldonderzoek door middel van boringen Twaalf apostelenweg te Nijmegen","description":[],"publicationdate":"2017-01-01","publisher":"Aeres Milieu","embargoenddate":"2017-07-06","source":[],"format":["application/pdf"],"contributor":["Feest, NJW Van Der","Aeres Milieu"],"coverage":[],"bestaccessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"50|datacite____::0a15c3ad876db2ffa2f8f1c9a63c3cd0","originalId":["datacite____::0a15c3ad876db2ffa2f8f1c9a63c3cd0"],"pid":[{"scheme":"doi","value":"10.17026/dans-24w-bckq"}],"dateofcollection":"","lastupdatetimestamp":1591283098682,"instance":[{"accessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Other literature type","url":["http://dx.doi.org/10.17026/dans-24w-bckq"],"publicationdate":"2017-01-01"}]} -{"author":[{"fullname":"Daas, Martinus","name":"Martinus","surname":"Daas","rank":1,"pid":null},{"fullname":"Weijer, Antonius Van De","name":"Antonius","surname":"Weijer","rank":2,"pid":null},{"fullname":"Vos, Willem De","name":"Willem","surname":"Vos","rank":3,"pid":null},{"fullname":"Oost, John Van Der","name":"John","surname":"Oost","rank":4,"pid":null},{"fullname":"Kranenburg, Richard Van","name":"Richard","surname":"Kranenburg","rank":5,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Microbiology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Genetics"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Molecular Biology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Biotechnology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"59999 Environmental Sciences not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"39999 Chemical Sciences not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Ecology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Immunology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Inorganic Chemistry"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Science Policy"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Plant Biology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"MOESM3 of Isolation of a genetically accessible thermophilic xylan degrading bacterium from compost","description":["Additional file 3: Table S2. HPLC data from isolates ranked on total organic acid production and total lactic acid production on cellobiose (C6)."],"publicationdate":"2016-01-01","publisher":"Figshare","source":[],"format":[],"contributor":[],"coverage":[],"id":"50|datacite____::150d96a57b37c02f5d14a6ace965d790","originalId":["datacite____::150d96a57b37c02f5d14a6ace965d790"],"pid":[{"scheme":"doi","value":"10.6084/m9.figshare.c.3603305_d2"}],"dateofcollection":"","lastupdatetimestamp":1591283087583,"instance":[{"license":"https://creativecommons.org/licenses/by/4.0","type":"Other literature type","url":["http://dx.doi.org/10.6084/m9.figshare.c.3603305_d2"],"publicationdate":"2016-01-01"}]} -{"author":[{"fullname":"Mosleh, Marwan","name":"Marwan","surname":"Mosleh","rank":1,"pid":null},{"fullname":"Jeesh, Yousef Al","name":"Yousef Al","surname":"Jeesh","rank":2,"pid":null},{"fullname":"Koustuv Dalal","name":"","surname":"","rank":3,"pid":null},{"fullname":"Charli Eriksson","name":"","surname":"","rank":4,"pid":null},{"fullname":"Carlerby, Heidi","name":"Heidi","surname":"Carlerby","rank":5,"pid":null},{"fullname":"Viitasara, Eija","name":"Eija","surname":"Viitasara","rank":6,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Medicine"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Ecology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Sociology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"69999 Biological Sciences not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Science Policy"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"acm","value":"Data_FILES"},"provenance":{"provenance":"iis","trust":"0.891"}}],"maintitle":"Additional file 1 of Barriers to managing and delivery of care to war-injured survivors or patients with non-communicable disease: a qualitative study of Palestinian patients’ and policy-makers’ perspectives","description":["Additional file 1. Study discussion guides."],"publicationdate":"2020-01-01","publisher":"figshare","source":[],"format":[],"contributor":[],"coverage":[],"id":"50|datacite____::1e099cf76219212d554ad35b45d2345c","originalId":["datacite____::1e099cf76219212d554ad35b45d2345c"],"pid":[{"scheme":"doi","value":"10.6084/m9.figshare.12285566"}],"dateofcollection":"","lastupdatetimestamp":1591283012211,"instance":[{"license":"https://creativecommons.org/licenses/by/4.0","type":"Other literature type","url":["http://dx.doi.org/10.6084/m9.figshare.12285566"],"publicationdate":"2020-01-01"}]} -{"author":[{"fullname":"Kuijl, E.E.A. Van Der","name":"E. E. A.","surname":"Kuijl","rank":1,"pid":null}],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"PROSPECTIE"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Inventariserend veldonderzoek d.m.v. boringen, Ittervoorterweg te Swartbroek","description":[],"publicationdate":"2009-01-01","publisher":"Synthegra","embargoenddate":"2009-11-27","source":[],"format":["25 p."],"contributor":["Janssens, M.","Hoeven, F. Van Der","Hensen, G.","Coolen, J.","Wijnen, J.","Synthegra Archeologie"],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"50|datacite____::24cce4f26db20e50803102c3c7961a8e","originalId":["datacite____::24cce4f26db20e50803102c3c7961a8e"],"pid":[{"scheme":"doi","value":"10.17026/dans-zwz-6cvc"}],"dateofcollection":"","lastupdatetimestamp":1591282632316,"instance":[{"license":"http://creativecommons.org/publicdomain/zero/1.0","accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Other literature type","url":["http://dx.doi.org/10.17026/dans-zwz-6cvc"],"publicationdate":"2009-01-01"}]} -{"author":[{"fullname":"Embree, Jennifer","name":"Jennifer","surname":"Embree","rank":1,"pid":null}],"type":"publication","language":{"code":"English","label":"English"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Destruction of cultural property"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Digital humanities"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Syria--History--Syrian Civil War, 2011-"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Archives by Any Other name: Archiving Memory During Times of Conflict through Non-Traditional Methods--A Case Study on Digital Humanities Projects Manar al-Athar and Project Syria","description":["Over the last century, conflicts across the world have resulted in an unprecedented number of cultural heritage sites being purposefully targeted for destruction. While there have been several historical attempts to combat this destruction, the emerging field of digital humanities is now using new digital technologies to also document and preserve cultural heritage demolishment. This article conducts case studies of two such projects: Project Syria, a virtual reality experience documenting the Syrian Civil War, and Manar al-Athar, a digital photo archive that collects pictures of cultural heritage sites in the Middle East. This exploratory study seeks to compare past methods of preservation and documentation of cultural heritage during times of conflict to current methods of preservation and documentation through digital humanities projects, and to determine what digital humanities projects can accomplish that more traditional methods of preservation cannot."],"publicationdate":"2018-01-01","publisher":"The University of North Carolina at Chapel Hill University Libraries","source":[],"format":[],"contributor":[],"coverage":[],"id":"50|datacite____::26243564100cd29b39382d2321372a95","originalId":["datacite____::26243564100cd29b39382d2321372a95"],"pid":[{"scheme":"doi","value":"10.17615/xh7w-qv18"}],"dateofcollection":"","lastupdatetimestamp":1591282702184,"instance":[{"type":"Other literature type","url":["http://dx.doi.org/10.17615/xh7w-qv18"],"publicationdate":"2018-01-01"}]} -{"author":[{"fullname":"Huber, Brigitte","name":"Brigitte","surname":"Huber","rank":1,"pid":null},{"fullname":"Barnidge, Matthew","name":"Matthew","surname":"Barnidge","rank":2,"pid":null},{"fullname":"Zúñiga, Homero Gil De","name":"Homero Gil","surname":"Zúñiga","rank":3,"pid":null},{"fullname":"Liu, James","name":"James","surname":"Liu","rank":4,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"200199 Communication and Media Studies not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Science Policy"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Supplemental_Material – Supplemental material for Fostering public trust in science: The role of social media","description":["Supplemental material, Supplemental_Material for Fostering public trust in science: The role of social media by Brigitte Huber, Matthew Barnidge, Homero Gil de Zúñiga and James Liu in Public Understanding of Science"],"publicationdate":"2019-01-01","publisher":"SAGE Journals","source":[],"format":[],"contributor":[],"coverage":[],"id":"50|datacite____::2d1773354e6c79eee7001407cd8da2f0","originalId":["datacite____::2d1773354e6c79eee7001407cd8da2f0"],"pid":[{"scheme":"doi","value":"10.25384/sage.9869183.v1"}],"dateofcollection":"","lastupdatetimestamp":1591282547342,"instance":[{"license":"https://creativecommons.org/licenses/by/4.0","type":"Other literature type","url":["http://dx.doi.org/10.25384/sage.9869183.v1"],"publicationdate":"2019-01-01"}]} -{"author":[{"fullname":"Шогенцукова Залина Хасановна","name":"","surname":"","rank":1,"pid":null},{"fullname":"Гедгафова Ирина Юрьевна","name":"","surname":"","rank":2,"pid":null},{"fullname":"Мирзоева Жанна Мухарбиевна","name":"","surname":"","rank":3,"pid":null},{"fullname":"Шогенцуков Али Хасанович","name":"","surname":"","rank":4,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"кластеры"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"урожайность в овощеводстве"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"селекция"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"современные технологии"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"«продовольственная безопасность»"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"АПК"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"растениеводство"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"животноводство"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"модель «тройной спирали»"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"модернизация"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"селекция."},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"clusters"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"productivity in vegetable growing"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"selection"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"modern technologies"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"“food security”"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"agriculture"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"crop production"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"animal husbandry"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"“triple helix” model"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"modernization"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"selection."},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"clusters"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"productivity in vegetable growing"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"selection"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"modern technologies"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"“food security”"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"agriculture"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"crop production"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"animal husbandry"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"“triple helix” model"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"modernization"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"selection."},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Кластеры как инструмент управления агробизнесом Кабардино-Балкарской Республики","description":["Статья посвящена исследованию понятия кластера и его использования как инструмента управления повышения эффективности деятельности агропромышленным комплексом Кабардино-Балкарской Республики. Рассматриваются предпосылки и особенности кластеризации АПК как в отдельном регионе, так и в России в целом. Реализация кластерной политики в области сельского хозяйства России является инновационным подходом развития отрасли и повышения конкурентоспособности производимой продукции на рынке, повышения эффективности производственного процесса и т.д. В статье исследована модель «тройной спирали», используемой при создании и функционировании кластеров в сфере АПК. Исследование кластеров, как инструмент управления АПК отдельного региона, в частности Кабардино-Балкарской Республики, позволяет выявить факторы, обуславливающие необходимость данного процесса с одной стороны, а также выявлять резервы и иные возможности для общего развития эффективности АПК России и активации внедрения инновационных механизмов в сельское хозяйство.","The Article is devoted to the study of the concept of cluster and their use as a management tool to improve the efficiency of the agro-industrial complex of the KabardinoBalkaria Republic. The prerequisites and features of agribusiness clustering both in a separate region and in Russia as a whole are considered. The implementations of the cluster policy in the field of agriculture in Russia are an innovative approach to the development of the industry and improve the competitiveness of products in the market, improve the efficiency of the production process, etc. The article investigates the model of “triple helix” used in the creation and functioning of clusters in the field of agriculture. The study of clusters as an instrument of agribusiness management in a particular region, in particular the Kabardino-Balkaria Republic, allows to identify the factors causing the need for this process on the one hand, as well as to identify reserves and other opportunities for the overall development of the efficiency of the Russian agribusiness and the activation of the introduction of innovative mechanisms in agriculture."],"publicationdate":"2019-01-01","publisher":"Московский экономический журнал","source":[],"format":[],"contributor":[],"coverage":[],"id":"50|datacite____::2fa3de5d0846180a43214310234e5526","originalId":["datacite____::2fa3de5d0846180a43214310234e5526"],"pid":[{"scheme":"doi","value":"10.24411/2413-046x-2019-16022"}],"dateofcollection":"","lastupdatetimestamp":1591283178985,"instance":[{"type":"Other literature type","url":["http://dx.doi.org/10.24411/2413-046x-2019-16022"],"publicationdate":"2019-01-01"}]} \ No newline at end of file +{"author":[{"fullname":"van Someren, Christian","name":"Christian","surname":"van Someren","rank":1,"pid":null}],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"energieproductie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Management"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Monitoring"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Policy and Law"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Energie interventies en gedrag"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"publieke ondersteuning en communicatie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Professional practice & society"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Energie opwek","description":["Over het Energieakkoord. In het energieakkoord voor duurzame groei is afgesproken dat in 2020 14 procent van de opwek hernieuwbaar moet zijn en in 2023 16 procent. De doelstelling is een uitdagende opgave waarbij de eerste vraag is: \"Hoeveel hernieuwbare energie wordt er op dit moment opgewekt in Nederland?\" Deze website geeft antwoord op de vraag voor de actueel opgewekte windenergie, zonne-energie en biogas."],"publicationdate":"2016-11-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"DansKnawCris::3c81248c335f0aa07e06817ece6fa6af","originalId":["DansKnawCris::3c81248c335f0aa07e06817ece6fa6af"],"pid":[{"scheme":"urn","value":"urn:nbn:nl:hs:18-813a5dfa-4fd0-44c4-8cbf-310324dc724d"},{"scheme":"urn","value":"urn:nbn:nl:hs:18-813a5dfa-4fd0-44c4-8cbf-310324dc724d"}],"dateofcollection":"","lastupdatetimestamp":1591282663379,"instance":[{"accessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Other literature type","url":["http://energieopwek.nl/"],"publicationdate":"2016-11-01"}]} +{"author":[],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Vlaardingen"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Plangebied Het Hof en Oranjepark : gemeente Vlaardingen : archeologisch vooronderzoek: een inventariserend veldonderzoek (verkennende fase)","description":["Met lit. opg"],"publicationdate":"2010-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"DansKnawCris::4669a378a73661417182c208e6fdab53","originalId":["DansKnawCris::4669a378a73661417182c208e6fdab53"],"pid":[{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceFullCatalogue&search=priref=800007467"},{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceFullCatalogue&search=priref=800007467"}],"dateofcollection":"","lastupdatetimestamp":1591282758835,"instance":[{"accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Report","publicationdate":"2010-01-01"}]} +{"author":[],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"prospectie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Honselersdijk tracé persleiding (gemeente Westland) : een bureauonderzoek en inventariserend veldonderzoek in de vorm van een verkennend en karterend booronderzoek","description":["Lit.opg."],"publicationdate":"2008-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"DansKnawCris::52c4541c9bffde34daa945ece8dcf635","originalId":["DansKnawCris::52c4541c9bffde34daa945ece8dcf635"],"pid":[{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550013915"},{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550013915"}],"dateofcollection":"","lastupdatetimestamp":1591283000091,"instance":[{"accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Report","publicationdate":"2008-01-01"}]} +{"author":[],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Archeologisch onderzoek plangebied Akker-Boekenderweg te Thorn","description":[],"publicationdate":"2012-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"DansKnawCris::794d07c2e66f1fbf07d61b9bfca36dc2","originalId":["DansKnawCris::794d07c2e66f1fbf07d61b9bfca36dc2"],"pid":[{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550028404"},{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550028404"}],"dateofcollection":"","lastupdatetimestamp":1591282847497,"instance":[{"accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Report","publicationdate":"2012-01-01"}]} +{"author":[{"fullname":"Zwieten, van, P.A.M.","name":"van, P.A.M.","surname":"Zwieten","rank":1,"pid":null},{"fullname":"Banda, M.","name":"M.","surname":"Banda","rank":2,"pid":null},{"fullname":"Kolding, J.","name":"J.","surname":"Kolding","rank":3,"pid":null}],"type":"publication","language":{"code":"en","label":"en"},"country":[],"subjects":[{"subject":{"scheme":"","value":"african great-lakes"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"cichlid fishes"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"reference points"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"east-africa"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"management"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"perspective"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"tanganyika"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"diversity"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"history"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"nyasa"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Selecting indicators to assess the fisheries of Lake Malawi and Lake Malombe: Knowledge base and evaluative capacity","description":["The provision of management information on the fisheries of Lakes Malawi and Malombe has been characterised by top–down controlled single species steady-state assessment techniques originating from single gear industrial fisheries but applied to an open access highly diverse and adaptive small-scale multispecies and multi-gear fishery. The result has largely been an unhappy marriage with uncertainties blamed more on the data than the process, although the data collection generally is detailed and comprehensive on catch and effort parameters. An extensive literature review of primary and grey literature on ecosystem drivers, exploitation pressures, and fish population and community states shows that Malawi has the necessary knowledge base for expanding their assessment into multi-causal and exploratory indicator-based methods that can assist in better understanding and more disciplined use of existing data and monitoring systems. Selection and ranking of a suite of indicators focusing on the major fisheries in the Southeast arm of Lake Malawi and Lake Malombe were done by a group of Malawian fisheries researchers and management advisers, thereby testing a framework of scoring criteria assessing an indicator's acceptability, observability, and relatedness to management. Indicators that are close to raw observational data and that require limited permutations and few assumptions appear to be preferable in the Malawian context. CPUE-based assessments can improve the utility of data and information in communicating developments and processes and evaluate fisheries management policies"],"publicationdate":"2011-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"container":{"name":"Journal of Great Lakes Research","issnPrinted":"0380-1330","issnOnline":"","issnLinking":"","ep":"44","iss":"1","sp":"26","vol":"37","edition":"","conferenceplace":null,"conferencedate":null},"id":"DansKnawCris::b90247718304c409331edb82fd0e8d56","originalId":["DansKnawCris::b90247718304c409331edb82fd0e8d56"],"pid":[{"scheme":"urn","value":"urn:nbn:nl:ui:32-401847"},{"scheme":"urn","value":"urn:nbn:nl:ui:32-401847"}],"dateofcollection":"","lastupdatetimestamp":1591282621858,"instance":[{"accessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Article","publicationdate":"2011-01-01"}]} +{"author":[],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"archeologie"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Bureauonderzoek"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Clavecymbelstraat te Maastricht","description":[],"publicationdate":"2010-01-01","source":[],"format":[],"contributor":[],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"DansKnawCris::ba8f1ea3adf1bba501ec9da3a58e9638","originalId":["DansKnawCris::ba8f1ea3adf1bba501ec9da3a58e9638"],"pid":[{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550019564"},{"scheme":"urn","value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550019564"}],"dateofcollection":"","lastupdatetimestamp":1591282723014,"instance":[{"accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Report","publicationdate":"2010-01-01"}]} +{"author":[{"fullname":"Wegwarth, Odette","name":"Odette","surname":"Wegwarth","rank":1,"pid":null},{"fullname":"Pashayan, Nora","name":"Nora","surname":"Pashayan","rank":2,"pid":null},{"fullname":"Widschwendter, Martin","name":"Martin","surname":"Widschwendter","rank":3,"pid":null},{"fullname":"Rebitschek, Felix","name":"Felix","surname":"Rebitschek","rank":4,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Medicine"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Molecular Biology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Sociology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"19999 Mathematical Sciences not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Developmental Biology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Cancer"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Science Policy"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Additional file 1: of Womenâ s perception, attitudes, and intended behavior towards predictive epigenetic risk testing for female cancers in 5 European countries: a cross-sectional online survey","description":["STROBE checklist. (DOC 98 kb)"],"publicationdate":"2019-01-01","publisher":"Figshare","source":[],"format":[],"contributor":[],"coverage":[],"id":"datacite____::016dfd6ecbfaa929bbd90ec3a2b51521","originalId":["datacite____::016dfd6ecbfaa929bbd90ec3a2b51521"],"pid":[{"scheme":"doi","value":"10.6084/m9.figshare.8207303.v1"}],"dateofcollection":"","lastupdatetimestamp":1591282908770,"instance":[{"license":"https://creativecommons.org/licenses/by/4.0","type":"Other literature type","url":["http://dx.doi.org/10.6084/m9.figshare.8207303.v1"],"publicationdate":"2019-01-01"}]} +{"author":[{"fullname":"Archaeology South East","name":"","surname":"","rank":1,"pid":null}],"type":"publication","language":{"code":"en","label":"en"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Grey Literature"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"An Archaeological Evaluation at 290 -294 Golder's Green Road, London Borough of Barnet NW11","description":[],"publicationdate":"2007-01-01","publisher":"Archaeology Data Service","source":[],"format":["PDF"],"contributor":[],"coverage":[],"id":"datacite____::05c611fdfc93d7a2a703d1324e28104a","originalId":["datacite____::05c611fdfc93d7a2a703d1324e28104a"],"pid":[{"scheme":"doi","value":"10.5284/1003453"}],"dateofcollection":"","lastupdatetimestamp":1591282746270,"instance":[{"type":"Other literature type","url":["http://dx.doi.org/10.5284/1003453"],"articleprocessingcharge":{"currency":"EUR","amount":"3131.64"},"publicationdate":"2007-01-01"}]} +{"author":[{"fullname":"Veen, V. Van Der","name":"V.","surname":"Veen","rank":1,"pid":null}],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"verkennend booronderzoek"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Onbekend (XXX)"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Archeologisch bureau- en verkennend veldonderzoek door middel van boringen Twaalf apostelenweg te Nijmegen","description":[],"publicationdate":"2017-01-01","publisher":"Aeres Milieu","embargoenddate":"2017-07-06","source":[],"format":["application/pdf"],"contributor":["Feest, NJW Van Der","Aeres Milieu"],"coverage":[],"bestaccessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"datacite____::0a15c3ad876db2ffa2f8f1c9a63c3cd0","originalId":["datacite____::0a15c3ad876db2ffa2f8f1c9a63c3cd0"],"pid":[{"scheme":"doi","value":"10.17026/dans-24w-bckq"}],"dateofcollection":"","lastupdatetimestamp":1591283098682,"instance":[{"accessright":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Other literature type","url":["http://dx.doi.org/10.17026/dans-24w-bckq"],"publicationdate":"2017-01-01"}]} +{"author":[{"fullname":"Daas, Martinus","name":"Martinus","surname":"Daas","rank":1,"pid":null},{"fullname":"Weijer, Antonius Van De","name":"Antonius","surname":"Weijer","rank":2,"pid":null},{"fullname":"Vos, Willem De","name":"Willem","surname":"Vos","rank":3,"pid":null},{"fullname":"Oost, John Van Der","name":"John","surname":"Oost","rank":4,"pid":null},{"fullname":"Kranenburg, Richard Van","name":"Richard","surname":"Kranenburg","rank":5,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Microbiology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Genetics"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Molecular Biology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Biotechnology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"59999 Environmental Sciences not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"39999 Chemical Sciences not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Ecology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Immunology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Inorganic Chemistry"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Science Policy"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Plant Biology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"MOESM3 of Isolation of a genetically accessible thermophilic xylan degrading bacterium from compost","description":["Additional file 3: Table S2. HPLC data from isolates ranked on total organic acid production and total lactic acid production on cellobiose (C6)."],"publicationdate":"2016-01-01","publisher":"Figshare","source":[],"format":[],"contributor":[],"coverage":[],"id":"datacite____::150d96a57b37c02f5d14a6ace965d790","originalId":["datacite____::150d96a57b37c02f5d14a6ace965d790"],"pid":[{"scheme":"doi","value":"10.6084/m9.figshare.c.3603305_d2"}],"dateofcollection":"","lastupdatetimestamp":1591283087583,"instance":[{"license":"https://creativecommons.org/licenses/by/4.0","type":"Other literature type","url":["http://dx.doi.org/10.6084/m9.figshare.c.3603305_d2"],"publicationdate":"2016-01-01"}]} +{"author":[{"fullname":"Mosleh, Marwan","name":"Marwan","surname":"Mosleh","rank":1,"pid":null},{"fullname":"Jeesh, Yousef Al","name":"Yousef Al","surname":"Jeesh","rank":2,"pid":null},{"fullname":"Koustuv Dalal","name":"","surname":"","rank":3,"pid":null},{"fullname":"Charli Eriksson","name":"","surname":"","rank":4,"pid":null},{"fullname":"Carlerby, Heidi","name":"Heidi","surname":"Carlerby","rank":5,"pid":null},{"fullname":"Viitasara, Eija","name":"Eija","surname":"Viitasara","rank":6,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Medicine"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Ecology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Sociology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"69999 Biological Sciences not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Science Policy"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"acm","value":"Data_FILES"},"provenance":{"provenance":"iis","trust":"0.891"}}],"maintitle":"Additional file 1 of Barriers to managing and delivery of care to war-injured survivors or patients with non-communicable disease: a qualitative study of Palestinian patients’ and policy-makers’ perspectives","description":["Additional file 1. Study discussion guides."],"publicationdate":"2020-01-01","publisher":"figshare","source":[],"format":[],"contributor":[],"coverage":[],"id":"datacite____::1e099cf76219212d554ad35b45d2345c","originalId":["datacite____::1e099cf76219212d554ad35b45d2345c"],"pid":[{"scheme":"doi","value":"10.6084/m9.figshare.12285566"}],"dateofcollection":"","lastupdatetimestamp":1591283012211,"instance":[{"license":"https://creativecommons.org/licenses/by/4.0","type":"Other literature type","url":["http://dx.doi.org/10.6084/m9.figshare.12285566"],"publicationdate":"2020-01-01"}]} +{"author":[{"fullname":"Kuijl, E.E.A. Van Der","name":"E. E. A.","surname":"Kuijl","rank":1,"pid":null}],"type":"publication","language":{"code":"nl","label":"nl"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Archaeology"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"PROSPECTIE"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Inventariserend veldonderzoek d.m.v. boringen, Ittervoorterweg te Swartbroek","description":[],"publicationdate":"2009-01-01","publisher":"Synthegra","embargoenddate":"2009-11-27","source":[],"format":["25 p."],"contributor":["Janssens, M.","Hoeven, F. Van Der","Hensen, G.","Coolen, J.","Wijnen, J.","Synthegra Archeologie"],"coverage":[],"bestaccessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"id":"datacite____::24cce4f26db20e50803102c3c7961a8e","originalId":["datacite____::24cce4f26db20e50803102c3c7961a8e"],"pid":[{"scheme":"doi","value":"10.17026/dans-zwz-6cvc"}],"dateofcollection":"","lastupdatetimestamp":1591282632316,"instance":[{"license":"http://creativecommons.org/publicdomain/zero/1.0","accessright":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Other literature type","url":["http://dx.doi.org/10.17026/dans-zwz-6cvc"],"publicationdate":"2009-01-01"}]} +{"author":[{"fullname":"Embree, Jennifer","name":"Jennifer","surname":"Embree","rank":1,"pid":null}],"type":"publication","language":{"code":"English","label":"English"},"country":[],"subjects":[{"subject":{"scheme":"","value":"Destruction of cultural property"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Digital humanities"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Syria--History--Syrian Civil War, 2011-"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Archives by Any Other name: Archiving Memory During Times of Conflict through Non-Traditional Methods--A Case Study on Digital Humanities Projects Manar al-Athar and Project Syria","description":["Over the last century, conflicts across the world have resulted in an unprecedented number of cultural heritage sites being purposefully targeted for destruction. While there have been several historical attempts to combat this destruction, the emerging field of digital humanities is now using new digital technologies to also document and preserve cultural heritage demolishment. This article conducts case studies of two such projects: Project Syria, a virtual reality experience documenting the Syrian Civil War, and Manar al-Athar, a digital photo archive that collects pictures of cultural heritage sites in the Middle East. This exploratory study seeks to compare past methods of preservation and documentation of cultural heritage during times of conflict to current methods of preservation and documentation through digital humanities projects, and to determine what digital humanities projects can accomplish that more traditional methods of preservation cannot."],"publicationdate":"2018-01-01","publisher":"The University of North Carolina at Chapel Hill University Libraries","source":[],"format":[],"contributor":[],"coverage":[],"id":"datacite____::26243564100cd29b39382d2321372a95","originalId":["datacite____::26243564100cd29b39382d2321372a95"],"pid":[{"scheme":"doi","value":"10.17615/xh7w-qv18"}],"dateofcollection":"","lastupdatetimestamp":1591282702184,"instance":[{"type":"Other literature type","url":["http://dx.doi.org/10.17615/xh7w-qv18"],"publicationdate":"2018-01-01"}]} +{"author":[{"fullname":"Huber, Brigitte","name":"Brigitte","surname":"Huber","rank":1,"pid":null},{"fullname":"Barnidge, Matthew","name":"Matthew","surname":"Barnidge","rank":2,"pid":null},{"fullname":"Zúñiga, Homero Gil De","name":"Homero Gil","surname":"Zúñiga","rank":3,"pid":null},{"fullname":"Liu, James","name":"James","surname":"Liu","rank":4,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"200199 Communication and Media Studies not elsewhere classified"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"Science Policy"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Supplemental_Material – Supplemental material for Fostering public trust in science: The role of social media","description":["Supplemental material, Supplemental_Material for Fostering public trust in science: The role of social media by Brigitte Huber, Matthew Barnidge, Homero Gil de Zúñiga and James Liu in Public Understanding of Science"],"publicationdate":"2019-01-01","publisher":"SAGE Journals","source":[],"format":[],"contributor":[],"coverage":[],"id":"datacite____::2d1773354e6c79eee7001407cd8da2f0","originalId":["datacite____::2d1773354e6c79eee7001407cd8da2f0"],"pid":[{"scheme":"doi","value":"10.25384/sage.9869183.v1"}],"dateofcollection":"","lastupdatetimestamp":1591282547342,"instance":[{"license":"https://creativecommons.org/licenses/by/4.0","type":"Other literature type","url":["http://dx.doi.org/10.25384/sage.9869183.v1"],"publicationdate":"2019-01-01"}]} +{"author":[{"fullname":"Шогенцукова Залина Хасановна","name":"","surname":"","rank":1,"pid":null},{"fullname":"Гедгафова Ирина Юрьевна","name":"","surname":"","rank":2,"pid":null},{"fullname":"Мирзоева Жанна Мухарбиевна","name":"","surname":"","rank":3,"pid":null},{"fullname":"Шогенцуков Али Хасанович","name":"","surname":"","rank":4,"pid":null}],"type":"publication","language":{"code":"UNKNOWN","label":"UNKNOWN"},"country":[],"subjects":[{"subject":{"scheme":"","value":"кластеры"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"урожайность в овощеводстве"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"селекция"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"современные технологии"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"«продовольственная безопасность»"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"АПК"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"растениеводство"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"животноводство"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"модель «тройной спирали»"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"модернизация"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"селекция."},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"clusters"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"productivity in vegetable growing"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"selection"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"modern technologies"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"“food security”"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"agriculture"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"crop production"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"animal husbandry"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"“triple helix” model"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"modernization"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"selection."},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"clusters"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"productivity in vegetable growing"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"selection"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"modern technologies"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"“food security”"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"agriculture"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"crop production"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"animal husbandry"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"“triple helix” model"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"modernization"},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}},{"subject":{"scheme":"","value":"selection."},"provenance":{"provenance":"sysimport:crosswalk:datasetarchive","trust":"0.9"}}],"maintitle":"Кластеры как инструмент управления агробизнесом Кабардино-Балкарской Республики","description":["Статья посвящена исследованию понятия кластера и его использования как инструмента управления повышения эффективности деятельности агропромышленным комплексом Кабардино-Балкарской Республики. Рассматриваются предпосылки и особенности кластеризации АПК как в отдельном регионе, так и в России в целом. Реализация кластерной политики в области сельского хозяйства России является инновационным подходом развития отрасли и повышения конкурентоспособности производимой продукции на рынке, повышения эффективности производственного процесса и т.д. В статье исследована модель «тройной спирали», используемой при создании и функционировании кластеров в сфере АПК. Исследование кластеров, как инструмент управления АПК отдельного региона, в частности Кабардино-Балкарской Республики, позволяет выявить факторы, обуславливающие необходимость данного процесса с одной стороны, а также выявлять резервы и иные возможности для общего развития эффективности АПК России и активации внедрения инновационных механизмов в сельское хозяйство.","The Article is devoted to the study of the concept of cluster and their use as a management tool to improve the efficiency of the agro-industrial complex of the KabardinoBalkaria Republic. The prerequisites and features of agribusiness clustering both in a separate region and in Russia as a whole are considered. The implementations of the cluster policy in the field of agriculture in Russia are an innovative approach to the development of the industry and improve the competitiveness of products in the market, improve the efficiency of the production process, etc. The article investigates the model of “triple helix” used in the creation and functioning of clusters in the field of agriculture. The study of clusters as an instrument of agribusiness management in a particular region, in particular the Kabardino-Balkaria Republic, allows to identify the factors causing the need for this process on the one hand, as well as to identify reserves and other opportunities for the overall development of the efficiency of the Russian agribusiness and the activation of the introduction of innovative mechanisms in agriculture."],"publicationdate":"2019-01-01","publisher":"Московский экономический журнал","source":[],"format":[],"contributor":[],"coverage":[],"id":"datacite____::2fa3de5d0846180a43214310234e5526","originalId":["datacite____::2fa3de5d0846180a43214310234e5526"],"pid":[{"scheme":"doi","value":"10.24411/2413-046x-2019-16022"}],"dateofcollection":"","lastupdatetimestamp":1591283178985,"instance":[{"type":"Other literature type","url":["http://dx.doi.org/10.24411/2413-046x-2019-16022"],"publicationdate":"2019-01-01"}]} \ No newline at end of file diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/working/relation/context b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/working/relation/context index 5a117c3..c881b17 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/working/relation/context +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/working/relation/context @@ -1,25 +1,25 @@ -{"source":{"id":"00|context_____::99c8ef576f385bc322564d5694df6fc2","type":"context"},"target":{"id":"10|doajarticles::9c4b678901e5276d9e3addee566816af","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|doajarticles::9c4b678901e5276d9e3addee566816af","type":"datasource"},"target":{"id":"00|context_____::99c8ef576f385bc322564d5694df6fc2","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::e15922110564cf669aaed346e871bc01","type":"context"},"target":{"id":"10|doajarticles::acb7c79bb85d3b3a7b75389f5d9570f5","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|doajarticles::acb7c79bb85d3b3a7b75389f5d9570f5","type":"datasource"},"target":{"id":"00|context_____::e15922110564cf669aaed346e871bc01","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","type":"context"},"target":{"id":"10|doajarticles::1fa6859d71faa77b32d82f278c6ed1df","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|doajarticles::1fa6859d71faa77b32d82f278c6ed1df","type":"datasource"},"target":{"id":"00|context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","type":"context"},"target":{"id":"10|doajarticles::6eb31d13b12bc06bbac06aef63cf33c9","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|doajarticles::6eb31d13b12bc06bbac06aef63cf33c9","type":"datasource"},"target":{"id":"00|context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"target":{"id":"40|aka_________::01bb7b48e29d732a1c7bc5150b9195c4","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"40|aka_________::01bb7b48e29d732a1c7bc5150b9195c4","type":"datasource"},"target":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"target":{"id":"40|aka_________::01bb7b48e29d732a1c7bc5150b9195c2","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"40|aka_________::01bb7b48e29d732a1c7bc5150b9195c2","type":"datasource"},"target":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"target":{"id":"10|openaire____::c5502a43e76feab55dd00cf50f519125","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|openaire____::c5502a43e76feab55dd00cf50f519125","type":"datasource"},"target":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"target":{"id":"10|re3data_____::a48f09c562b247a9919acfe195549b47","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|re3data_____::a48f09c562b247a9919acfe195549b47","type":"datasource"},"target":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"target":{"id":"10|opendoar____::97275a23ca44226c9964043c8462be96","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|opendoar____::97275a23ca44226c9964043c8462be96","type":"datasource"},"target":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"target":{"id":"10|doajarticles::2899208a99aa7d142646e0a80bfeef05","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|doajarticles::2899208a99aa7d142646e0a80bfeef05","type":"datasource"},"target":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::e6c151d449e1db05b1ffb5ad5ec656cf","type":"context"},"target":{"id":"10|re3data_____::5b9bf9171d92df854cf3c520692e9122","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|re3data_____::5b9bf9171d92df854cf3c520692e9122","type":"datasource"},"target":{"id":"00|context_____::e6c151d449e1db05b1ffb5ad5ec656cf","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::e6c151d449e1db05b1ffb5ad5ec656cf","type":"context"},"target":{"id":"10|doajarticles::c7d3de67dc77af72f6747157441252ec","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"10|doajarticles::c7d3de67dc77af72f6747157441252ec","type":"datasource"},"target":{"id":"00|context_____::e6c151d449e1db05b1ffb5ad5ec656cf","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::e6c151d449e1db05b1ffb5ad5ec656cf","type":"context"},"target":{"id":"10|re3data_____::8515794670370f49c1d176c399c714f5","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} \ No newline at end of file +{"source":"context_____::99c8ef576f385bc322564d5694df6fc2","sourceType":"context","target":"doajarticles::9c4b678901e5276d9e3addee566816af","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"doajarticles::9c4b678901e5276d9e3addee566816af","sourceType":"datasource","target":"context_____::99c8ef576f385bc322564d5694df6fc2","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::e15922110564cf669aaed346e871bc01","sourceType":"context","target":"doajarticles::acb7c79bb85d3b3a7b75389f5d9570f5","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"doajarticles::acb7c79bb85d3b3a7b75389f5d9570f5","sourceType":"datasource","target":"context_____::e15922110564cf669aaed346e871bc01","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","sourceType":"context","target":"doajarticles::1fa6859d71faa77b32d82f278c6ed1df","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"doajarticles::1fa6859d71faa77b32d82f278c6ed1df","sourceType":"datasource","target":"context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","sourceType":"context","target":"doajarticles::6eb31d13b12bc06bbac06aef63cf33c9","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"doajarticles::6eb31d13b12bc06bbac06aef63cf33c9","sourceType":"datasource","target":"context_____::aa0e56dd2e9d2a0be749f5debdd2b3d8","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::04a00617ca659adc944977ac700ea14b","sourceType":"context","target":"aka_________::01bb7b48e29d732a1c7bc5150b9195c4","targetType":"project","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"aka_________::01bb7b48e29d732a1c7bc5150b9195c4","sourceType":"project","target":"context_____::04a00617ca659adc944977ac700ea14b","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::04a00617ca659adc944977ac700ea14b","sourceType":"context","target":"aka_________::01bb7b48e29d732a1c7bc5150b9195c2","targetType":"project","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"aka_________::01bb7b48e29d732a1c7bc5150b9195c2","sourceType":"project","target":"context_____::04a00617ca659adc944977ac700ea14b","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::04a00617ca659adc944977ac700ea14b","sourceType":"context","target":"openaire____::c5502a43e76feab55dd00cf50f519125","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"openaire____::c5502a43e76feab55dd00cf50f519125","sourceType":"datasource","target":"context_____::04a00617ca659adc944977ac700ea14b","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::04a00617ca659adc944977ac700ea14b","sourceType":"context","target":"re3data_____::a48f09c562b247a9919acfe195549b47","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"re3data_____::a48f09c562b247a9919acfe195549b47","sourceType":"datasource","target":"context_____::04a00617ca659adc944977ac700ea14b","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::04a00617ca659adc944977ac700ea14b","sourceType":"context","target":"opendoar____::97275a23ca44226c9964043c8462be96","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"opendoar____::97275a23ca44226c9964043c8462be96","sourceType":"datasource","target":"context_____::04a00617ca659adc944977ac700ea14b","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::04a00617ca659adc944977ac700ea14b","sourceType":"context","target":"doajarticles::2899208a99aa7d142646e0a80bfeef05","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"doajarticles::2899208a99aa7d142646e0a80bfeef05","sourceType":"datasource","target":"context_____::04a00617ca659adc944977ac700ea14b","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::e6c151d449e1db05b1ffb5ad5ec656cf","sourceType":"context","target":"re3data_____::5b9bf9171d92df854cf3c520692e9122","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"re3data_____::5b9bf9171d92df854cf3c520692e9122","sourceType":"datasource","target":"context_____::e6c151d449e1db05b1ffb5ad5ec656cf","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::e6c151d449e1db05b1ffb5ad5ec656cf","sourceType":"context","target":"doajarticles::c7d3de67dc77af72f6747157441252ec","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"doajarticles::c7d3de67dc77af72f6747157441252ec","sourceType":"datasource","target":"context_____::e6c151d449e1db05b1ffb5ad5ec656cf","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::e6c151d449e1db05b1ffb5ad5ec656cf","sourceType":"context","target":"re3data_____::8515794670370f49c1d176c399c714f5","targetType":"datasource","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} \ No newline at end of file diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/working/relation/contextOrg b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/working/relation/contextOrg index 8654fce..3f0beba 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/working/relation/contextOrg +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/subset/working/relation/contextOrg @@ -1,6 +1,6 @@ -{"source":{"id":"20|grid________::87698402476531ba39e61f1df38f2a91","type":"datasource"},"target":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::04a00617ca659adc944977ac700ea14b","type":"context"},"target":{"id":"20|grid________::87698402476531ba39e61f1df38f2a91","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"20|grid________::94948cc036605bf4a00ec77ce5ca92d3","type":"datasource"},"target":{"id":"00|context_____::5fde864866ea5ded4cc873b3170b63c3","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::5fde864866ea5ded4cc873b3170b63c3","type":"context"},"target":{"id":"20|grid________::94948cc036605bf4a00ec77ce5ca92d3","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"20|grid________::94948cc036605bf4a00ec77ce5ca92d3","type":"datasource"},"target":{"id":"00|context_____::e6c151d449e1db05b1ffb5ad5ec656cf","type":"context"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} -{"source":{"id":"00|context_____::e6c151d449e1db05b1ffb5ad5ec656cf","type":"context"},"target":{"id":"20|grid________::94948cc036605bf4a00ec77ce5ca92d3","type":"datasource"},"reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} \ No newline at end of file +{"source":"grid________::87698402476531ba39e61f1df38f2a91","sourceType":"organization","target":"context_____::04a00617ca659adc944977ac700ea14b","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::04a00617ca659adc944977ac700ea14b","sourceType":"context","target":"grid________::87698402476531ba39e61f1df38f2a91","targetType":"organization","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"grid________::94948cc036605bf4a00ec77ce5ca92d3","sourceType":"organization","target":"context_____::5fde864866ea5ded4cc873b3170b63c3","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::5fde864866ea5ded4cc873b3170b63c3","sourceType":"context","target":"grid________::94948cc036605bf4a00ec77ce5ca92d3","targetType":"organization","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"grid________::94948cc036605bf4a00ec77ce5ca92d3","sourceType":"organization","target":"context_____::e6c151d449e1db05b1ffb5ad5ec656cf","targetType":"context","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} +{"source":"context_____::e6c151d449e1db05b1ffb5ad5ec656cf","sourceType":"context","target":"grid________::94948cc036605bf4a00ec77ce5ca92d3","targetType":"organization","reltype":{"name":"IsRelatedTo","type":"relationship"},"provenance":{"provenance":"Linked by user","trust":"0.9"},"validated":false,"validationDate":null} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7f650b4..c889688 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ 5.6.1 3.5 11.0.2 - [2.12.1] + [3.16.0] \ No newline at end of file From 2e8639f22d94ec38421ade2d4019f8253a1ff94b Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 1 Jun 2023 15:10:00 +0200 Subject: [PATCH 02/12] added test to verify the dump for indicators at the level of project and datasource. Fixed issue on identifier with the prefix --- .../eu/dnetlib/dhp/oa/model/UsageCounts.java | 4 +- .../eu/dnetlib/dhp/oa/model/graph/Node.java | 38 ----------- .../dnetlib/dhp/oa/model/graph/Project.java | 8 +++ .../dhp/oa/graph/dump/complete/Extractor.java | 2 +- .../dump/complete/SparkDumpEntitiesJob.java | 9 ++- .../dump/complete/SparkDumpRelationJob.java | 2 +- .../complete/SparkOrganizationRelation.java | 2 +- ...DumpOrganizationProjectDatasourceTest.java | 67 ++++++++++++++----- .../oa/graph/dump/subset/DumpSubsetTest.java | 3 +- .../dump/complete/datasource/datasource.json | 4 +- .../graph/dump/complete/project/project.json | 4 +- 11 files changed, 76 insertions(+), 67 deletions(-) delete mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Node.java diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/UsageCounts.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/UsageCounts.java index 21cf602..2f894e0 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/UsageCounts.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/UsageCounts.java @@ -1,11 +1,13 @@ package eu.dnetlib.dhp.oa.model; +import java.io.Serializable; + /** * @author miriam.baglioni * @Date 07/11/22 */ -public class UsageCounts { +public class UsageCounts implements Serializable { private String downloads; private String views; diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Node.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Node.java deleted file mode 100644 index cd2bb78..0000000 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Node.java +++ /dev/null @@ -1,38 +0,0 @@ - -package eu.dnetlib.dhp.oa.model.graph; - -import java.io.Serializable; - -/** - * To represent the generic node in a relation. It has the following parameters: - private String id the openaire id of - * the entity in the relation - private String type the type of the entity in the relation. Consider the generic - * relation between a Result R and a Project P, the node representing R will have as id the id of R and as type result, - * while the node representing the project will have as id the id of the project and as type project - */ -public class Node implements Serializable { - private String id; - private String type; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public static Node newInstance(String id, String type) { - Node node = new Node(); - node.id = id; - node.type = type; - return node; - } -} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java index 6480c87..8b44f6b 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java @@ -73,6 +73,14 @@ public class Project implements Serializable { @JsonSchema(description = "Indicators computed for this project, for example UsageCount ones") private Indicator indicators; + public Indicator getIndicators() { + return indicators; + } + + public void setIndicators(Indicator indicators) { + this.indicators = indicators; + } + public String getId() { return id; } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java index 5bcc571..8e18fec 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java @@ -15,7 +15,7 @@ import org.apache.spark.sql.SparkSession; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.model.Provenance; -import eu.dnetlib.dhp.oa.model.graph.Node; + import eu.dnetlib.dhp.oa.model.graph.RelType; import eu.dnetlib.dhp.oa.model.graph.Relation; import eu.dnetlib.dhp.schema.common.ModelConstants; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java index 6f9a0e0..47b4ee7 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java @@ -213,7 +213,7 @@ public class SparkDumpEntitiesJob implements Serializable { private static Datasource mapDatasource(eu.dnetlib.dhp.schema.oaf.Datasource d) { Datasource datasource = new Datasource(); - datasource.setId(d.getId()); + datasource.setId(d.getId().substring(3)); Optional .ofNullable(d.getOriginalId()) @@ -337,6 +337,9 @@ public class SparkDumpEntitiesJob implements Serializable { .ofNullable(d.getJournal()) .ifPresent(j -> datasource.setJournal(getContainer(j))); + Optional.ofNullable(d.getMeasures()) + .ifPresent(m -> datasource.setIndicators(Utils.getIndicator(d.getMeasures()))); + return datasource; } @@ -503,7 +506,7 @@ public class SparkDumpEntitiesJob implements Serializable { project.setFunding(funList); if (Optional.ofNullable(p.getMeasures()).isPresent()) { - + project.setIndicators(Utils.getIndicator(p.getMeasures())); } return project; } @@ -605,7 +608,7 @@ public class SparkDumpEntitiesJob implements Serializable { Optional .ofNullable(org.getId()) - .ifPresent(value -> organization.setId(value)); + .ifPresent(value -> organization.setId(value.substring(3))); Optional .ofNullable(org.getPid()) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java index 5a2c073..e14cdbd 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java @@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.model.Provenance; -import eu.dnetlib.dhp.oa.model.graph.Node; + import eu.dnetlib.dhp.oa.model.graph.RelType; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.DataInfo; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java index 7a0750f..c947664 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java @@ -24,7 +24,7 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.model.Provenance; -import eu.dnetlib.dhp.oa.model.graph.Node; + import eu.dnetlib.dhp.oa.model.graph.RelType; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java index eda1556..c01d081 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java @@ -6,12 +6,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; +import eu.dnetlib.dhp.oa.model.Indicator; import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.ForeachFunction; import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -97,10 +99,11 @@ public class DumpOrganizationProjectDatasourceTest { Assertions.assertEquals(15, verificationDataset.count()); - verificationDataset - .foreach( - (ForeachFunction) o -> System.out - .println(OBJECT_MAPPER.writeValueAsString(o))); + //TODO write significant assertions +// verificationDataset +// .foreach( +// (ForeachFunction) o -> System.out +// .println(OBJECT_MAPPER.writeValueAsString(o))); } @@ -132,10 +135,25 @@ public class DumpOrganizationProjectDatasourceTest { Assertions.assertEquals(12, verificationDataset.count()); - verificationDataset - .foreach( - (ForeachFunction) o -> System.out - .println(OBJECT_MAPPER.writeValueAsString(o))); + Assertions.assertEquals(10, verificationDataset.filter("indicators is NULL").count()); + Assertions.assertEquals(2, verificationDataset.filter("indicators is not NULL").count()); + Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'aka_________::01bb7b48e29d732a1c7bc5150b9195c4'").count()); + Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'aka_________::9d1af21dbd0f5bc719f71553d19a6b3a'").count()); + + eu.dnetlib.dhp.oa.model.graph.Project p = tmp.filter(pr -> pr.getId().equals("aka_________::01bb7b48e29d732a1c7bc5150b9195c4")).first(); + Assertions.assertEquals("2019",p.getIndicators().getUsageCounts().getDownloads()); + Assertions.assertEquals("1804",p.getIndicators().getUsageCounts().getViews()); + Assertions.assertNull(p.getIndicators().getImpactMeasures()); + + p = tmp.filter(pr -> pr.getId().equals("aka_________::9d1af21dbd0f5bc719f71553d19a6b3a")).first(); + Assertions.assertEquals("139",p.getIndicators().getUsageCounts().getDownloads()); + Assertions.assertEquals("53",p.getIndicators().getUsageCounts().getViews()); + Assertions.assertNull(p.getIndicators().getImpactMeasures()); + //TODO write significant assertions +// verificationDataset +// .foreach( +// (ForeachFunction) o -> System.out +// .println(OBJECT_MAPPER.writeValueAsString(o))); } @@ -166,10 +184,26 @@ public class DumpOrganizationProjectDatasourceTest { Assertions.assertEquals(5, verificationDataset.count()); - verificationDataset - .foreach( - (ForeachFunction) o -> System.out - .println(OBJECT_MAPPER.writeValueAsString(o))); + Assertions.assertEquals(3, verificationDataset.filter("indicators is NULL").count()); + Assertions.assertEquals(2, verificationDataset.filter("indicators is not NULL").count()); + Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'doajarticles::1fa6859d71faa77b32d82f278c6ed1df'").count()); + Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'doajarticles::9c4b678901e5276d9e3addee566816af'").count()); + + eu.dnetlib.dhp.oa.model.graph.Datasource p = tmp.filter(pr -> pr.getId().equals("doajarticles::1fa6859d71faa77b32d82f278c6ed1df")).first(); + Assertions.assertEquals("47542",p.getIndicators().getUsageCounts().getDownloads()); + Assertions.assertEquals("36485",p.getIndicators().getUsageCounts().getViews()); + Assertions.assertNull(p.getIndicators().getImpactMeasures()); + + p = tmp.filter(pr -> pr.getId().equals("doajarticles::9c4b678901e5276d9e3addee566816af")).first(); + Assertions.assertEquals("981357",p.getIndicators().getUsageCounts().getDownloads()); + Assertions.assertEquals("646539",p.getIndicators().getUsageCounts().getViews()); + Assertions.assertNull(p.getIndicators().getImpactMeasures()); + + //TODO write significant assertions +// verificationDataset +// .foreach( +// (ForeachFunction) o -> System.out +// .println(OBJECT_MAPPER.writeValueAsString(o))); } @Test @@ -199,10 +233,11 @@ public class DumpOrganizationProjectDatasourceTest { Assertions.assertEquals(1, verificationDataset.count()); - verificationDataset - .foreach( - (ForeachFunction) o -> System.out - .println(OBJECT_MAPPER.writeValueAsString(o))); + //TODO write significant assertions +// verificationDataset +// .foreach( +// (ForeachFunction) o -> System.out +// .println(OBJECT_MAPPER.writeValueAsString(o))); } } diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/subset/DumpSubsetTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/subset/DumpSubsetTest.java index a9dcb5e..cdfa2f3 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/subset/DumpSubsetTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/subset/DumpSubsetTest.java @@ -472,7 +472,6 @@ public class DumpSubsetTest { .textFile(workingDir.toString() + "/dump/relation") .map(item -> OBJECT_MAPPER.readValue(item, eu.dnetlib.dhp.oa.model.graph.Relation.class)); - Assertions.assertEquals(10, tmp.count()); Assertions.assertEquals(5, tmp.filter(r -> r.getSourceType().equals("context")).count()); @@ -515,7 +514,7 @@ public class DumpSubsetTest { Assertions.assertEquals(102, tmp.count()); - Assertions.assertEquals(51, tmp.filter(r -> r.getSourceType().equals("result") ).count()); + Assertions.assertEquals(51, tmp.filter(r -> r.getSourceType().equals("result")).count()); Assertions.assertEquals(39, tmp.filter(r -> r.getSourceType().equals("datasource")).count()); Assertions.assertEquals(12, tmp.filter(r -> r.getSourceType().equals("context")).count()); } diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/datasource/datasource.json b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/datasource/datasource.json index 882ca0e..87f587c 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/datasource/datasource.json +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/datasource/datasource.json @@ -1,5 +1,5 @@ -{"accessinfopackage":[],"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dataprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"datasourcetype":{"classid":"pubsrepository::journal","classname":"Journal","schemeid":"dnet:datasource_typologies","schemename":"dnet:datasource_typologies"},"dateofcollection":"2018-06-05","englishname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal of Applied Mathematics and Stochastic Analysis"},"extraInfo":[],"id":"10|doajarticles::1fa6859d71faa77b32d82f278c6ed1df","lastupdatetimestamp":1592688952862,"latitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"longitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"namespaceprefix":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"doaj10489533"},"odcontenttypes":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal articles"}],"odlanguages":[],"odnumberofitems":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"officialname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal of Applied Mathematics and Stochastic Analysis"},"openairecompatibility":{"classid":"hostedBy","classname":"collected from a compatible aggregator","schemeid":"dnet:datasourceCompatibilityLevel","schemename":"dnet:datasourceCompatibilityLevel"},"originalId":["doajarticles::1048-9533",null],"pid":[],"policies":[],"serviceprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"subjects":[],"versioning":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"websiteurl":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"https://www.hindawi.com/journals/jamsa"}} -{"accessinfopackage":[],"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dataprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"datasourcetype":{"classid":"pubsrepository::journal","classname":"Journal","schemeid":"dnet:datasource_typologies","schemename":"dnet:datasource_typologies"},"dateofcollection":"2020-05-25","englishname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Pelitutkimuksen vuosikirja"},"extraInfo":[],"id":"10|doajarticles::9c4b678901e5276d9e3addee566816af","lastupdatetimestamp":1592688952862,"latitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"longitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"namespaceprefix":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"doaj1798355X"},"odcontenttypes":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal articles"}],"odlanguages":[],"odnumberofitems":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"officialname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Pelitutkimuksen vuosikirja"},"openairecompatibility":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:datasourceCompatibilityLevel","schemename":"dnet:datasourceCompatibilityLevel"},"originalId":["doajarticles::1798-355X",null],"pid":[],"policies":[],"serviceprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"subjects":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Geography. Anthropology. Recreation: Recreation. Leisure | Science: Mathematics: Instruments and machines: Electronic computers. Computer science: Computer software"}],"versioning":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"websiteurl":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"http://www.pelitutkimus.fi"}} +{"measures":[{"id":"downloads","unit":[{"key":"count","value":"47542","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"views","unit":[{"key":"count","value":"36485","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]}],"accessinfopackage":[],"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dataprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"datasourcetype":{"classid":"pubsrepository::journal","classname":"Journal","schemeid":"dnet:datasource_typologies","schemename":"dnet:datasource_typologies"},"dateofcollection":"2018-06-05","englishname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal of Applied Mathematics and Stochastic Analysis"},"extraInfo":[],"id":"10|doajarticles::1fa6859d71faa77b32d82f278c6ed1df","lastupdatetimestamp":1592688952862,"latitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"longitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"namespaceprefix":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"doaj10489533"},"odcontenttypes":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal articles"}],"odlanguages":[],"odnumberofitems":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"officialname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal of Applied Mathematics and Stochastic Analysis"},"openairecompatibility":{"classid":"hostedBy","classname":"collected from a compatible aggregator","schemeid":"dnet:datasourceCompatibilityLevel","schemename":"dnet:datasourceCompatibilityLevel"},"originalId":["doajarticles::1048-9533",null],"pid":[],"policies":[],"serviceprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"subjects":[],"versioning":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"websiteurl":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"https://www.hindawi.com/journals/jamsa"}} +{"measures":[{"id":"downloads","unit":[{"key":"count","value":"981357","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"views","unit":[{"key":"count","value":"646539","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]}],"accessinfopackage":[],"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dataprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"datasourcetype":{"classid":"pubsrepository::journal","classname":"Journal","schemeid":"dnet:datasource_typologies","schemename":"dnet:datasource_typologies"},"dateofcollection":"2020-05-25","englishname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Pelitutkimuksen vuosikirja"},"extraInfo":[],"id":"10|doajarticles::9c4b678901e5276d9e3addee566816af","lastupdatetimestamp":1592688952862,"latitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"longitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"namespaceprefix":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"doaj1798355X"},"odcontenttypes":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal articles"}],"odlanguages":[],"odnumberofitems":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"officialname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Pelitutkimuksen vuosikirja"},"openairecompatibility":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:datasourceCompatibilityLevel","schemename":"dnet:datasourceCompatibilityLevel"},"originalId":["doajarticles::1798-355X",null],"pid":[],"policies":[],"serviceprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"subjects":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Geography. Anthropology. Recreation: Recreation. Leisure | Science: Mathematics: Instruments and machines: Electronic computers. Computer science: Computer software"}],"versioning":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"websiteurl":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"http://www.pelitutkimus.fi"}} {"accessinfopackage":[],"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dataprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"datasourcetype":{"classid":"pubsrepository::journal","classname":"Journal","schemeid":"dnet:datasource_typologies","schemename":"dnet:datasource_typologies"},"dateofcollection":"2018-06-05","englishname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Statistika: Statistics and Economy Journal"},"extraInfo":[],"id":"10|doajarticles::a5314b60f79b869cb5d3a2709167bc3a","lastupdatetimestamp":1592688952862,"latitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"longitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"namespaceprefix":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"doaj0322788X"},"odcontenttypes":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal articles"}],"odlanguages":[],"odnumberofitems":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"officialname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Statistika: Statistics and Economy Journal"},"openairecompatibility":{"classid":"hostedBy","classname":"collected from a compatible aggregator","schemeid":"dnet:datasourceCompatibilityLevel","schemename":"dnet:datasourceCompatibilityLevel"},"originalId":["doajarticles::0322-788X",null],"pid":[],"policies":[],"serviceprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"subjects":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Social Sciences: Statistics"}],"versioning":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"websiteurl":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"http://www.czso.cz/statistika_journal"}} {"accessinfopackage":[],"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dataprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"datasourcetype":{"classid":"pubsrepository::journal","classname":"Journal","schemeid":"dnet:datasource_typologies","schemename":"dnet:datasource_typologies"},"dateofcollection":"2018-06-05","englishname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Review of Development Finance"},"extraInfo":[],"id":"10|doajarticles::acb7c79bb85d3b3a7b75389f5d9570f5","lastupdatetimestamp":1592688952862,"latitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"longitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"namespaceprefix":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"doaj18799337"},"odcontenttypes":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Journal articles"}],"odlanguages":[],"odnumberofitems":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"officialname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Review of Development Finance"},"openairecompatibility":{"classid":"hostedBy","classname":"collected from a compatible aggregator","schemeid":"dnet:datasourceCompatibilityLevel","schemename":"dnet:datasourceCompatibilityLevel"},"originalId":["doajarticles::1879-9337",null],"pid":[],"policies":[],"serviceprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"subjects":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Social Sciences: Industries. Land use. Labor: Economic growth, development, planning | Social Sciences: Finance"}],"versioning":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"websiteurl":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"http://www.journals.elsevier.com/review-of-development-finance/"}} {"accessinfopackage":[],"collectedfrom":[{"key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dataprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"datasourcetype":{"classid":"pubsrepository::journal","classname":"Journal","schemeid":"dnet:datasource_typologies","schemename":"dnet:datasource_typologies"},"dateofcollection":"2020-05-28","englishname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"The Journal of Advanced Navigation Technology"},"extraInfo":[],"id":"10|issn___print::0a79337eaf5145faa478785423273355","lastupdatetimestamp":1592688952862,"latitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"longitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"namespaceprefix":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"jrnl12269026"},"odcontenttypes":[],"odlanguages":[],"odnumberofitems":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"0.0"},"officialname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"The Journal of Advanced Navigation Technology"},"openairecompatibility":{"classid":"hostedBy","classname":"collected from a compatible aggregator","schemeid":"dnet:datasourceCompatibilityLevel","schemename":"dnet:datasourceCompatibilityLevel"},"originalId":["issn___print::1226-9026",null],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":null,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"qualifier":{"classid":"re3data","classname":"re3data","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"r3d100012161"}],"policies":[],"serviceprovider":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false},"subjects":[],"versioning":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":false}} \ No newline at end of file diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/project/project.json b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/project/project.json index 61a9125..c2b7f4e 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/project/project.json +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/project/project.json @@ -1,5 +1,5 @@ -{"callidentifier":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Fotoniikka ja modernit kuvantamismenetelmät LT"},"code":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"135027"},"collectedfrom":[{"key":"10|openaire____::6ac933301a3933c8a22ceebea7000326","value":"Academy of Finland"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"dateofcollection":"2019-01-25","dateoftransformation":"2019-04-16","duration":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"0"},"ecarticle29_3":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"ecsc39":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"extraInfo":[],"fundedamount":0.0,"fundingtree":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"\n aka_________::AKA\n AKA\n Academy of Finland\n Academy of Finland\n FI\n "}],"id":"40|aka_________::01bb7b48e29d732a1c7bc5150b9195c4","jsonextrainfo":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"{}"},"lastupdatetimestamp":1594398578323,"oamandatepublications":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"optional1":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"190,000 €"},"originalId":["aka_________::135027"],"pid":[],"subjects":[],"title":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Dynamic 3D resolution-enhanced low-coherence interferometric imaging / Consortium: Hi-Lo"},"totalcost":0.0} -{"callidentifier":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Academy Project Funding TT"},"code":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"316061"},"collectedfrom":[{"key":"10|openaire____::6ac933301a3933c8a22ceebea7000326","value":"Academy of Finland"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"dateofcollection":"2019-01-25","dateoftransformation":"2019-04-16","duration":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"0"},"ecarticle29_3":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"ecsc39":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"extraInfo":[],"fundedamount":0.0,"fundingtree":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"\n aka_________::AKA\n AKA\n Academy of Finland\n Academy of Finland\n FI\n "}],"id":"40|aka_________::9d1af21dbd0f5bc719f71553d19a6b3a","jsonextrainfo":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"{}"},"lastupdatetimestamp":1594398578323,"oamandatepublications":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"optional1":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"450,000 €"},"originalId":["aka_________::316061"],"pid":[],"subjects":[],"title":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Finnish Imaging of Degenerative Shoulder Study (FIMAGE): A study on the prevalence of degenerative imaging changes of the shoulder and their relevance to clinical symptoms in the general population."},"totalcost":0.0} +{"measures":[{"id":"downloads","unit":[{"key":"count","value":"2019","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"views","unit":[{"key":"count","value":"1804","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]}],"callidentifier":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Fotoniikka ja modernit kuvantamismenetelmät LT"},"code":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"135027"},"collectedfrom":[{"key":"10|openaire____::6ac933301a3933c8a22ceebea7000326","value":"Academy of Finland"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"dateofcollection":"2019-01-25","dateoftransformation":"2019-04-16","duration":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"0"},"ecarticle29_3":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"ecsc39":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"extraInfo":[],"fundedamount":0.0,"fundingtree":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"\n aka_________::AKA\n AKA\n Academy of Finland\n Academy of Finland\n FI\n "}],"id":"40|aka_________::01bb7b48e29d732a1c7bc5150b9195c4","jsonextrainfo":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"{}"},"lastupdatetimestamp":1594398578323,"oamandatepublications":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"optional1":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"190,000 €"},"originalId":["aka_________::135027"],"pid":[],"subjects":[],"title":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Dynamic 3D resolution-enhanced low-coherence interferometric imaging / Consortium: Hi-Lo"},"totalcost":0.0} +{"measures":[{"id":"downloads","unit":[{"key":"count","value":"139","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"views","unit":[{"key":"count","value":"53","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]}],"callidentifier":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Academy Project Funding TT"},"code":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"316061"},"collectedfrom":[{"key":"10|openaire____::6ac933301a3933c8a22ceebea7000326","value":"Academy of Finland"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"dateofcollection":"2019-01-25","dateoftransformation":"2019-04-16","duration":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"0"},"ecarticle29_3":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"ecsc39":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"extraInfo":[],"fundedamount":0.0,"fundingtree":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"\n aka_________::AKA\n AKA\n Academy of Finland\n Academy of Finland\n FI\n "}],"id":"40|aka_________::9d1af21dbd0f5bc719f71553d19a6b3a","jsonextrainfo":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"{}"},"lastupdatetimestamp":1594398578323,"oamandatepublications":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"optional1":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"450,000 €"},"originalId":["aka_________::316061"],"pid":[],"subjects":[],"title":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Finnish Imaging of Degenerative Shoulder Study (FIMAGE): A study on the prevalence of degenerative imaging changes of the shoulder and their relevance to clinical symptoms in the general population."},"totalcost":0.0} {"acronym":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"MOISE"},"code":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"ANR-17-CE05-0033"},"collectedfrom":[{"key":"10|openaire____::457528c43fabd74e212db2ed61101075","value":"Agence Nationale de la Recherche"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"dateofcollection":"2019-12-24","dateoftransformation":"2020-01-07","duration":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"0"},"ecarticle29_3":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"ecsc39":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"extraInfo":[],"fundedamount":0.0,"fundingtree":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"\n anr_________::ANR\n ANR\n French National Research Agency (ANR)\n Agence Nationale de la Recherche\n FR\n "}],"id":"40|anr_________::1f21edc5c902be305ee47148955c6e50","jsonextrainfo":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"{}"},"lastupdatetimestamp":1594398578323,"oamandatepublications":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"originalId":["anr_________::ANR-17-CE05-0033"],"pid":[],"subjects":[],"title":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"METAL OXIDES AS LOW LOADED NANO-IRIDIUM SUPPORT FOR COMPETITIVE WATER ELECTROLYSIS"},"totalcost":0.0} {"acronym":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"GALAXY"},"code":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"ANR-09-SEGI-0005"},"collectedfrom":[{"key":"10|openaire____::457528c43fabd74e212db2ed61101075","value":"Agence Nationale de la Recherche"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"dateofcollection":"2019-12-24","dateoftransformation":"2020-01-07","duration":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"0"},"ecarticle29_3":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"ecsc39":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"extraInfo":[],"fundedamount":0.0,"fundingtree":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"\n anr_________::ANR\n ANR\n French National Research Agency (ANR)\n Agence Nationale de la Recherche\n FR\n "}],"id":"40|anr_________::547e78ffdcb7d72a1ef31058dede3a33","jsonextrainfo":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"{}"},"lastupdatetimestamp":1594398578323,"oamandatepublications":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"originalId":["anr_________::ANR-09-SEGI-0005"],"pid":[],"subjects":[],"title":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"DEVELOPPEMENT COLLABORATIF DE SYSTEMES COMPLEXES SELON UNE APPROCHE GUIDEE PAR LES MODELES"},"totalcost":0.0} {"code":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"LE0347462"},"collectedfrom":[{}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"dateofcollection":"2015-08-24","dateoftransformation":"2018-11-20","duration":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"0"},"ecarticle29_3":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"ecsc39":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"enddate":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"2003-12-31"},"extraInfo":[],"fundedamount":0.0,"fundingtree":[{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"\n \n arc_________::ARC\n ARC\n Australian Research Council (ARC)\n AU\n \n \n arc_________::ARC::Linkage Infrastructure, Equipment and Facilities\n Linkage Infrastructure, Equipment and Facilities\n Linkage Infrastructure, Equipment and Facilities\n \n arc:fundingStream\n \n "}],"id":"40|arc_________::838e781a8d479e27a11101421fd8b296","jsonextrainfo":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"{}"},"keywords":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"biomedical nanostructures,femtosecond laser machining,laser manufacturing,laser micromachining,microphotonics,photonic bandgap structures"},"lastupdatetimestamp":1594398578323,"oamandatepublications":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"false"},"originalId":["arc_________::LE0347462"],"pid":[],"startdate":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"2003-01-01"},"subjects":[],"title":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"Femtosecond laser micromachining facility"},"totalcost":0.0,"websiteurl":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900000000000000022"},"value":"http://purl.org/au-research/grants/arc/LE0347462"}} From 8661bc0c90a499d476c59253c9e7bb79d8234d6a Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 2 Jun 2023 16:13:18 +0200 Subject: [PATCH 03/12] aligned with the last version of pom for production --- .../src/test/java/GenerateJsonSchema.java | 7 +-- .../dhp/oa/graph/dump/complete/Extractor.java | 1 - .../dump/complete/SparkDumpEntitiesJob.java | 5 +- .../dump/complete/SparkDumpRelationJob.java | 1 - .../complete/SparkOrganizationRelation.java | 1 - ...DumpOrganizationProjectDatasourceTest.java | 62 +++++++++++++------ pom.xml | 2 +- 7 files changed, 49 insertions(+), 30 deletions(-) diff --git a/dump-schema/src/test/java/GenerateJsonSchema.java b/dump-schema/src/test/java/GenerateJsonSchema.java index 1dd03f8..dc4e4ff 100644 --- a/dump-schema/src/test/java/GenerateJsonSchema.java +++ b/dump-schema/src/test/java/GenerateJsonSchema.java @@ -1,5 +1,6 @@ import java.io.IOException; +import eu.dnetlib.dhp.oa.model.graph.*; import org.junit.jupiter.api.Test; import com.fasterxml.jackson.core.JsonProcessingException; @@ -10,10 +11,6 @@ import com.github.victools.jsonschema.generator.*; import eu.dnetlib.dhp.ExecCreateSchemas; import eu.dnetlib.dhp.oa.model.community.CommunityResult; -import eu.dnetlib.dhp.oa.model.graph.Datasource; -import eu.dnetlib.dhp.oa.model.graph.GraphResult; -import eu.dnetlib.dhp.oa.model.graph.Organization; -import eu.dnetlib.dhp.oa.model.graph.ResearchCommunity; //@Disabled class GenerateJsonSchema { @@ -44,7 +41,7 @@ class GenerateJsonSchema { .without(Option.NONPUBLIC_NONSTATIC_FIELDS_WITHOUT_GETTERS); SchemaGeneratorConfig config = configBuilder.build(); SchemaGenerator generator = new SchemaGenerator(config); - JsonNode jsonSchema = generator.generateSchema(GraphResult.class); + JsonNode jsonSchema = generator.generateSchema(Project.class); System.out.println(jsonSchema.toString()); } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java index 8e18fec..fd2712f 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java @@ -15,7 +15,6 @@ import org.apache.spark.sql.SparkSession; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.model.Provenance; - import eu.dnetlib.dhp.oa.model.graph.RelType; import eu.dnetlib.dhp.oa.model.graph.Relation; import eu.dnetlib.dhp.schema.common.ModelConstants; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java index 47b4ee7..bcf3f36 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java @@ -337,8 +337,9 @@ public class SparkDumpEntitiesJob implements Serializable { .ofNullable(d.getJournal()) .ifPresent(j -> datasource.setJournal(getContainer(j))); - Optional.ofNullable(d.getMeasures()) - .ifPresent(m -> datasource.setIndicators(Utils.getIndicator(d.getMeasures()))); + Optional + .ofNullable(d.getMeasures()) + .ifPresent(m -> datasource.setIndicators(Utils.getIndicator(d.getMeasures()))); return datasource; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java index e14cdbd..8aeb72d 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java @@ -23,7 +23,6 @@ import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.model.Provenance; - import eu.dnetlib.dhp.oa.model.graph.RelType; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.DataInfo; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java index c947664..473633b 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java @@ -24,7 +24,6 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.model.Provenance; - import eu.dnetlib.dhp.oa.model.graph.RelType; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java index c01d081..0c72649 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java @@ -6,7 +6,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; -import eu.dnetlib.dhp.oa.model.Indicator; import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; @@ -25,6 +24,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException; +import eu.dnetlib.dhp.oa.model.Indicator; import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Organization; import eu.dnetlib.dhp.schema.oaf.Project; @@ -99,7 +99,7 @@ public class DumpOrganizationProjectDatasourceTest { Assertions.assertEquals(15, verificationDataset.count()); - //TODO write significant assertions + // TODO write significant assertions // verificationDataset // .foreach( // (ForeachFunction) o -> System.out @@ -137,19 +137,31 @@ public class DumpOrganizationProjectDatasourceTest { Assertions.assertEquals(10, verificationDataset.filter("indicators is NULL").count()); Assertions.assertEquals(2, verificationDataset.filter("indicators is not NULL").count()); - Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'aka_________::01bb7b48e29d732a1c7bc5150b9195c4'").count()); - Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'aka_________::9d1af21dbd0f5bc719f71553d19a6b3a'").count()); + Assertions + .assertEquals( + 1, + verificationDataset + .filter("indicators is not NULL AND id == 'aka_________::01bb7b48e29d732a1c7bc5150b9195c4'") + .count()); + Assertions + .assertEquals( + 1, + verificationDataset + .filter("indicators is not NULL AND id == 'aka_________::9d1af21dbd0f5bc719f71553d19a6b3a'") + .count()); - eu.dnetlib.dhp.oa.model.graph.Project p = tmp.filter(pr -> pr.getId().equals("aka_________::01bb7b48e29d732a1c7bc5150b9195c4")).first(); - Assertions.assertEquals("2019",p.getIndicators().getUsageCounts().getDownloads()); - Assertions.assertEquals("1804",p.getIndicators().getUsageCounts().getViews()); + eu.dnetlib.dhp.oa.model.graph.Project p = tmp + .filter(pr -> pr.getId().equals("aka_________::01bb7b48e29d732a1c7bc5150b9195c4")) + .first(); + Assertions.assertEquals("2019", p.getIndicators().getUsageCounts().getDownloads()); + Assertions.assertEquals("1804", p.getIndicators().getUsageCounts().getViews()); Assertions.assertNull(p.getIndicators().getImpactMeasures()); p = tmp.filter(pr -> pr.getId().equals("aka_________::9d1af21dbd0f5bc719f71553d19a6b3a")).first(); - Assertions.assertEquals("139",p.getIndicators().getUsageCounts().getDownloads()); - Assertions.assertEquals("53",p.getIndicators().getUsageCounts().getViews()); + Assertions.assertEquals("139", p.getIndicators().getUsageCounts().getDownloads()); + Assertions.assertEquals("53", p.getIndicators().getUsageCounts().getViews()); Assertions.assertNull(p.getIndicators().getImpactMeasures()); - //TODO write significant assertions + // TODO write significant assertions // verificationDataset // .foreach( // (ForeachFunction) o -> System.out @@ -186,20 +198,32 @@ public class DumpOrganizationProjectDatasourceTest { Assertions.assertEquals(3, verificationDataset.filter("indicators is NULL").count()); Assertions.assertEquals(2, verificationDataset.filter("indicators is not NULL").count()); - Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'doajarticles::1fa6859d71faa77b32d82f278c6ed1df'").count()); - Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'doajarticles::9c4b678901e5276d9e3addee566816af'").count()); + Assertions + .assertEquals( + 1, + verificationDataset + .filter("indicators is not NULL AND id == 'doajarticles::1fa6859d71faa77b32d82f278c6ed1df'") + .count()); + Assertions + .assertEquals( + 1, + verificationDataset + .filter("indicators is not NULL AND id == 'doajarticles::9c4b678901e5276d9e3addee566816af'") + .count()); - eu.dnetlib.dhp.oa.model.graph.Datasource p = tmp.filter(pr -> pr.getId().equals("doajarticles::1fa6859d71faa77b32d82f278c6ed1df")).first(); - Assertions.assertEquals("47542",p.getIndicators().getUsageCounts().getDownloads()); - Assertions.assertEquals("36485",p.getIndicators().getUsageCounts().getViews()); + eu.dnetlib.dhp.oa.model.graph.Datasource p = tmp + .filter(pr -> pr.getId().equals("doajarticles::1fa6859d71faa77b32d82f278c6ed1df")) + .first(); + Assertions.assertEquals("47542", p.getIndicators().getUsageCounts().getDownloads()); + Assertions.assertEquals("36485", p.getIndicators().getUsageCounts().getViews()); Assertions.assertNull(p.getIndicators().getImpactMeasures()); p = tmp.filter(pr -> pr.getId().equals("doajarticles::9c4b678901e5276d9e3addee566816af")).first(); - Assertions.assertEquals("981357",p.getIndicators().getUsageCounts().getDownloads()); - Assertions.assertEquals("646539",p.getIndicators().getUsageCounts().getViews()); + Assertions.assertEquals("981357", p.getIndicators().getUsageCounts().getDownloads()); + Assertions.assertEquals("646539", p.getIndicators().getUsageCounts().getViews()); Assertions.assertNull(p.getIndicators().getImpactMeasures()); - //TODO write significant assertions + // TODO write significant assertions // verificationDataset // .foreach( // (ForeachFunction) o -> System.out @@ -233,7 +257,7 @@ public class DumpOrganizationProjectDatasourceTest { Assertions.assertEquals(1, verificationDataset.count()); - //TODO write significant assertions + // TODO write significant assertions // verificationDataset // .foreach( // (ForeachFunction) o -> System.out diff --git a/pom.xml b/pom.xml index c889688..ee4ec11 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ 5.6.1 3.5 11.0.2 - [3.16.0] + [2.13.1-patched] \ No newline at end of file From 95125d704ad8970516e619080849ea69893d8c7f Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 11 Jul 2023 12:40:21 +0200 Subject: [PATCH 04/12] [dump] removed usage stats info from the datasource and project --- .../dhp/oa/model/graph/Datasource.java | 20 ++++++------- .../dnetlib/dhp/oa/model/graph/Project.java | 20 ++++++------- .../dhp/oa/graph/dump/ResultMapper.java | 28 +++++++++---------- .../dump/complete/SparkDumpEntitiesJob.java | 12 ++++---- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Datasource.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Datasource.java index 7154287..7b21379 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Datasource.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Datasource.java @@ -129,16 +129,16 @@ public class Datasource implements Serializable { @JsonSchema(description = "Information about the journal, if this data source is of type Journal.") private Container journal; // issn etc del Journal - @JsonSchema(description = "Indicators computed for this Datasource, for example UsageCount ones") - private Indicator indicators; - - public Indicator getIndicators() { - return indicators; - } - - public void setIndicators(Indicator indicators) { - this.indicators = indicators; - } +// @JsonSchema(description = "Indicators computed for this Datasource, for example UsageCount ones") +// private Indicator indicators; +// +// public Indicator getIndicators() { +// return indicators; +// } +// +// public void setIndicators(Indicator indicators) { +// this.indicators = indicators; +// } public String getId() { return id; diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java index 8b44f6b..6ef3a5c 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Project.java @@ -70,16 +70,16 @@ public class Project implements Serializable { @JsonSchema(description = "The h2020 programme funding the project") private List h2020programme; - @JsonSchema(description = "Indicators computed for this project, for example UsageCount ones") - private Indicator indicators; - - public Indicator getIndicators() { - return indicators; - } - - public void setIndicators(Indicator indicators) { - this.indicators = indicators; - } +// @JsonSchema(description = "Indicators computed for this project, for example UsageCount ones") +// private Indicator indicators; +// +// public Indicator getIndicators() { +// return indicators; +// } +// +// public void setIndicators(Indicator indicators) { +// this.indicators = indicators; +// } public String getId() { return id; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java index dd01252..0814c68 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java @@ -173,20 +173,20 @@ public class ResultMapper implements Serializable { .ifPresent( value -> value .stream() - .filter( - s -> !((s.getQualifier().getClassid().equalsIgnoreCase("fos") && - Optional.ofNullable(s.getDataInfo()).isPresent() - && Optional.ofNullable(s.getDataInfo().getProvenanceaction()).isPresent() && - s.getDataInfo().getProvenanceaction().getClassid().equalsIgnoreCase("subject:fos")) - || - (s.getQualifier().getClassid().equalsIgnoreCase("sdg") && - Optional.ofNullable(s.getDataInfo()).isPresent() - && Optional.ofNullable(s.getDataInfo().getProvenanceaction()).isPresent() && - s - .getDataInfo() - .getProvenanceaction() - .getClassid() - .equalsIgnoreCase("subject:sdg")))) +// .filter( +// s -> !((s.getQualifier().getClassid().equalsIgnoreCase("fos") && +// Optional.ofNullable(s.getDataInfo()).isPresent() +// && Optional.ofNullable(s.getDataInfo().getProvenanceaction()).isPresent() && +// s.getDataInfo().getProvenanceaction().getClassid().equalsIgnoreCase("subject:fos")) +// || +// (s.getQualifier().getClassid().equalsIgnoreCase("sdg") && +// Optional.ofNullable(s.getDataInfo()).isPresent() +// && Optional.ofNullable(s.getDataInfo().getProvenanceaction()).isPresent() && +// s +// .getDataInfo() +// .getProvenanceaction() +// .getClassid() +// .equalsIgnoreCase("subject:sdg")))) .forEach(s -> subjectList.add(getSubject(s)))); out.setSubjects(subjectList); diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java index bcf3f36..2ac7bf4 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java @@ -337,9 +337,9 @@ public class SparkDumpEntitiesJob implements Serializable { .ofNullable(d.getJournal()) .ifPresent(j -> datasource.setJournal(getContainer(j))); - Optional - .ofNullable(d.getMeasures()) - .ifPresent(m -> datasource.setIndicators(Utils.getIndicator(d.getMeasures()))); +// Optional +// .ofNullable(d.getMeasures()) +// .ifPresent(m -> datasource.setIndicators(Utils.getIndicator(d.getMeasures()))); return datasource; @@ -506,9 +506,9 @@ public class SparkDumpEntitiesJob implements Serializable { } project.setFunding(funList); - if (Optional.ofNullable(p.getMeasures()).isPresent()) { - project.setIndicators(Utils.getIndicator(p.getMeasures())); - } +// if (Optional.ofNullable(p.getMeasures()).isPresent()) { +// project.setIndicators(Utils.getIndicator(p.getMeasures())); +// } return project; } From 787d4d0b4a77810ed1fa209b6cc3d3a7691c3838 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 13 Jul 2023 18:19:12 +0200 Subject: [PATCH 05/12] changed the pom reference to the dho schema --- .../dnetlib/dhp/oa/model/graph/Relation.java | 4 -- .../src/test/java/GenerateJsonSchema.java | 2 +- ...DumpOrganizationProjectDatasourceTest.java | 42 +++++++++---------- pom.xml | 2 +- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Relation.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Relation.java index ceb0339..2cc4280 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Relation.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/graph/Relation.java @@ -72,10 +72,6 @@ public class Relation implements Serializable { this.targetType = targetType; } - public boolean isValidated() { - return validated; - } - public RelType getReltype() { return reltype; } diff --git a/dump-schema/src/test/java/GenerateJsonSchema.java b/dump-schema/src/test/java/GenerateJsonSchema.java index dc4e4ff..35d20ff 100644 --- a/dump-schema/src/test/java/GenerateJsonSchema.java +++ b/dump-schema/src/test/java/GenerateJsonSchema.java @@ -1,6 +1,5 @@ import java.io.IOException; -import eu.dnetlib.dhp.oa.model.graph.*; import org.junit.jupiter.api.Test; import com.fasterxml.jackson.core.JsonProcessingException; @@ -11,6 +10,7 @@ import com.github.victools.jsonschema.generator.*; import eu.dnetlib.dhp.ExecCreateSchemas; import eu.dnetlib.dhp.oa.model.community.CommunityResult; +import eu.dnetlib.dhp.oa.model.graph.*; //@Disabled class GenerateJsonSchema { diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java index 0c72649..58fae3c 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java @@ -150,17 +150,17 @@ public class DumpOrganizationProjectDatasourceTest { .filter("indicators is not NULL AND id == 'aka_________::9d1af21dbd0f5bc719f71553d19a6b3a'") .count()); - eu.dnetlib.dhp.oa.model.graph.Project p = tmp - .filter(pr -> pr.getId().equals("aka_________::01bb7b48e29d732a1c7bc5150b9195c4")) - .first(); - Assertions.assertEquals("2019", p.getIndicators().getUsageCounts().getDownloads()); - Assertions.assertEquals("1804", p.getIndicators().getUsageCounts().getViews()); - Assertions.assertNull(p.getIndicators().getImpactMeasures()); +// eu.dnetlib.dhp.oa.model.graph.Project p = tmp +// .filter(pr -> pr.getId().equals("aka_________::01bb7b48e29d732a1c7bc5150b9195c4")) +// .first(); +// Assertions.assertEquals("2019", p.getIndicators().getUsageCounts().getDownloads()); +// Assertions.assertEquals("1804", p.getIndicators().getUsageCounts().getViews()); +// Assertions.assertNull(p.getIndicators().getImpactMeasures()); - p = tmp.filter(pr -> pr.getId().equals("aka_________::9d1af21dbd0f5bc719f71553d19a6b3a")).first(); - Assertions.assertEquals("139", p.getIndicators().getUsageCounts().getDownloads()); - Assertions.assertEquals("53", p.getIndicators().getUsageCounts().getViews()); - Assertions.assertNull(p.getIndicators().getImpactMeasures()); +// p = tmp.filter(pr -> pr.getId().equals("aka_________::9d1af21dbd0f5bc719f71553d19a6b3a")).first(); +// Assertions.assertEquals("139", p.getIndicators().getUsageCounts().getDownloads()); +// Assertions.assertEquals("53", p.getIndicators().getUsageCounts().getViews()); +// Assertions.assertNull(p.getIndicators().getImpactMeasures()); // TODO write significant assertions // verificationDataset // .foreach( @@ -211,17 +211,17 @@ public class DumpOrganizationProjectDatasourceTest { .filter("indicators is not NULL AND id == 'doajarticles::9c4b678901e5276d9e3addee566816af'") .count()); - eu.dnetlib.dhp.oa.model.graph.Datasource p = tmp - .filter(pr -> pr.getId().equals("doajarticles::1fa6859d71faa77b32d82f278c6ed1df")) - .first(); - Assertions.assertEquals("47542", p.getIndicators().getUsageCounts().getDownloads()); - Assertions.assertEquals("36485", p.getIndicators().getUsageCounts().getViews()); - Assertions.assertNull(p.getIndicators().getImpactMeasures()); - - p = tmp.filter(pr -> pr.getId().equals("doajarticles::9c4b678901e5276d9e3addee566816af")).first(); - Assertions.assertEquals("981357", p.getIndicators().getUsageCounts().getDownloads()); - Assertions.assertEquals("646539", p.getIndicators().getUsageCounts().getViews()); - Assertions.assertNull(p.getIndicators().getImpactMeasures()); +// eu.dnetlib.dhp.oa.model.graph.Datasource p = tmp +// .filter(pr -> pr.getId().equals("doajarticles::1fa6859d71faa77b32d82f278c6ed1df")) +// .first(); +// Assertions.assertEquals("47542", p.getIndicators().getUsageCounts().getDownloads()); +// Assertions.assertEquals("36485", p.getIndicators().getUsageCounts().getViews()); +// Assertions.assertNull(p.getIndicators().getImpactMeasures()); +// +// p = tmp.filter(pr -> pr.getId().equals("doajarticles::9c4b678901e5276d9e3addee566816af")).first(); +// Assertions.assertEquals("981357", p.getIndicators().getUsageCounts().getDownloads()); +// Assertions.assertEquals("646539", p.getIndicators().getUsageCounts().getViews()); +// Assertions.assertNull(p.getIndicators().getImpactMeasures()); // TODO write significant assertions // verificationDataset diff --git a/pom.xml b/pom.xml index ee4ec11..00b5e97 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ 5.6.1 3.5 11.0.2 - [2.13.1-patched] + [3.17.1] \ No newline at end of file From 21a521b97cf9510afa65f282011d1f449d5c7abb Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Sat, 15 Jul 2023 10:36:54 +0200 Subject: [PATCH 06/12] changed the API to consider the upload only of an already open version --- .../dhp/oa/graph/dump/SendToZenodoHDFS.java | 6 +- .../dhp/oa/zenodoapi/ZenodoAPIClient.java | 71 +++++++++---------- .../dhp/oa/graph/dump/ZenodoUploadTest.java | 38 +++++++--- 3 files changed, 64 insertions(+), 51 deletions(-) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java index 32031eb..bf90722 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java @@ -90,9 +90,9 @@ public class SendToZenodoHDFS implements Serializable { zenodoApiClient.sendMretadata(metadata); } - if (Boolean.TRUE.equals(publish)) { - zenodoApiClient.publish(); - } +// if (Boolean.TRUE.equals(publish)) { +// zenodoApiClient.publish(); +// } } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java index 1b3bb7a..d37be74 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java @@ -18,7 +18,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -//import org.apache.http.impl.client.HttpClients; +// import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.jetbrains.annotations.NotNull; @@ -108,11 +108,11 @@ public class ZenodoAPIClient implements Serializable { RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON); Request request = new Request.Builder() - .url(urlString) - .addHeader("Content-Type", "application/json") // add request headers - .addHeader("Authorization", "Bearer " + access_token) - .post(body) - .build(); + .url(urlString) + .addHeader("Content-Type", "application/json") // add request headers + .addHeader("Authorization", "Bearer " + access_token) + .post(body) + .build(); try (Response response = httpClient.newCall(request).execute()) { @@ -122,7 +122,8 @@ public class ZenodoAPIClient implements Serializable { // Get response body json = response.body().string(); - eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson().fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class); + eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson() + .fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class); this.bucket = newSubmission.getLinks().getBucket(); this.deposition_id = newSubmission.getId(); @@ -132,7 +133,6 @@ public class ZenodoAPIClient implements Serializable { } - // public int uploadIS2(InputStream is, String fileName) throws IOException { // // final String crlf = "\r\n"; @@ -340,10 +340,10 @@ public class ZenodoAPIClient implements Serializable { RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON); Request request = new Request.Builder() - .url(urlString + "/" + deposition_id + "/actions/newversion") - .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) - .post(body) - .build(); + .url(urlString + "/" + deposition_id + "/actions/newversion") + .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) + .post(body) + .build(); try (Response response = httpClient.newCall(request).execute()) { @@ -374,32 +374,25 @@ public class ZenodoAPIClient implements Serializable { this.deposition_id = deposition_id; - String json = "{}"; + OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build(); - URL url = new URL(urlString + "/" + deposition_id); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + Request request = new Request.Builder() + .url(urlString + "/" + deposition_id) + .addHeader("Authorization", "Bearer " + access_token) + .build(); + + try (Response response = httpClient.newCall(request).execute()) { + + if (!response.isSuccessful()) + throw new IOException("Unexpected code " + response + response.body().string()); + + eu.dnetlib.dhp.common.api.zenodo.ZenodoModel zenodoModel = new Gson() + .fromJson(response.body().string(), eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class); + bucket = zenodoModel.getLinks().getBucket(); + return response.code(); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setRequestMethod("POST"); - conn.setDoOutput(true); - try (OutputStream os = conn.getOutputStream()) { - byte[] input = json.getBytes("utf-8"); - os.write(input, 0, input.length); } - String body = getBody(conn); - - int responseCode = conn.getResponseCode(); - conn.disconnect(); - - if (!checkOKStatus(responseCode)) - throw new IOException("Unexpected code " + responseCode + body); - - ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); - bucket = zenodoModel.getLinks().getBucket(); - - return responseCode; - } private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException { @@ -445,11 +438,11 @@ public class ZenodoAPIClient implements Serializable { String url = urlBuilder.build().toString(); Request request = new Request.Builder() - .url(url) - .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers - .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) - .get() - .build(); + .url(url) + .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers + .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) + .get() + .build(); try (Response response = httpClient.newCall(request).execute()) { diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java index f119274..25aec29 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java @@ -7,8 +7,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; -import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient; -import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.junit.jupiter.api.Assertions; @@ -19,15 +17,16 @@ import org.junit.jupiter.api.Test; import com.google.gson.Gson; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; - +import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException; +import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient; @Disabled public class ZenodoUploadTest { private static String workingDir; - private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions"; - private final String ACCESS_TOKEN = "OzzOsyucEIHxCEfhlpsMo3myEiwpCza3trCRL7ddfGTAK9xXkIP2MbXd6Vg4"; + private final String URL_STRING = "https://zenodo.org/api/deposit/depositions"; + private final String ACCESS_TOKEN = "GxqutB1JnEmdvBafQI2cCjtUvoOs0novDuie3hxCEQUJcErHVMhkJjawIqhb"; @BeforeAll public static void beforeAll() throws IOException { @@ -86,8 +85,7 @@ public class ZenodoUploadTest { System.out.println(client.sendMretadata(metadata)); - - //System.out.println(client.publish()); + // System.out.println(client.publish()); } @@ -138,7 +136,7 @@ public class ZenodoUploadTest { } - //System.out.println(client.publish()); + // System.out.println(client.publish()); } @@ -146,7 +144,7 @@ public class ZenodoUploadTest { void testNewVersion2() throws Exception, MissingConceptDoiException { ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, - ACCESS_TOKEN); + ACCESS_TOKEN); client.newVersion("1210237"); @@ -162,6 +160,7 @@ public class ZenodoUploadTest { // Assertions.assertEquals(202, client.publish()); } + @Test void readCommunityMap() throws IOException { LocalFileSystem fs = FileSystem.getLocal(new Configuration()); @@ -195,4 +194,25 @@ public class ZenodoUploadTest { // Assertions.assertEquals(202, client.publish()); } + @Test + void testOnlyUpload() throws Exception, MissingConceptDoiException { + + ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, + ACCESS_TOKEN); + + client.uploadOpenDeposition("8144316"); + + File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar"); +// File file = new File(getClass() +// .getResource("/eu/dnetlib/dhp/common/api/newVersion2") +// .getPath()); + + InputStream is = new FileInputStream(file); + + Assertions.assertEquals(200, client.uploadIS3(is, "newVersion_deposition", file.length())); + + // Assertions.assertEquals(202, client.publish()); + + } + } From 6b113961c15299dbfb8752a65095bd7a184980ac Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 28 Jul 2023 10:26:22 +0200 Subject: [PATCH 07/12] - --- .../dhp/oa/graph/dump/ResultMapper.java | 6 ++--- .../eu/dnetlib/dhp/oa/graph/dump/Utils.java | 2 +- .../community/SparkPrepareResultProject.java | 19 +++++++++++---- .../dhp/oa/graph/dump/complete/Extractor.java | 4 ++-- .../dhp/oa/graph/dump/complete/Process.java | 6 ++--- .../dump/complete/SparkCollectAndSave.java | 24 ++++++++++--------- .../dump/complete/SparkDumpRelationJob.java | 4 +++- .../complete/SparkOrganizationRelation.java | 4 ++-- .../graph/dump/project_prep_parameters.json | 6 +++++ .../funder/oozie_app/workflow.xml | 1 + 10 files changed, 49 insertions(+), 27 deletions(-) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java index 0814c68..52584f9 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java @@ -155,7 +155,7 @@ public class ResultMapper implements Serializable { input .getCollectedfrom() .stream() - .map(cf -> CfHbKeyValue.newInstance(cf.getKey(), cf.getValue())) + .map(cf -> CfHbKeyValue.newInstance(cf.getKey().substring(3), cf.getValue())) .collect(Collectors.toList())); } @@ -518,11 +518,11 @@ public class ResultMapper implements Serializable { instance .setCollectedfrom( CfHbKeyValue - .newInstance(i.getCollectedfrom().getKey(), i.getCollectedfrom().getValue())); + .newInstance(i.getCollectedfrom().getKey().substring(3), i.getCollectedfrom().getValue())); instance .setHostedby( - CfHbKeyValue.newInstance(i.getHostedby().getKey(), i.getHostedby().getValue())); + CfHbKeyValue.newInstance(i.getHostedby().getKey().substring(3), i.getHostedby().getValue())); return instance; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java index dca42f4..b3f380e 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java @@ -63,7 +63,7 @@ public class Utils { return String .format( - "%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX, + "%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX, DHPUtils.md5(id)); } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkPrepareResultProject.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkPrepareResultProject.java index 079e708..8c4faba 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkPrepareResultProject.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkPrepareResultProject.java @@ -61,6 +61,13 @@ public class SparkPrepareResultProject implements Serializable { .orElse(Boolean.TRUE); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + Boolean substring = Optional + .ofNullable(parser.get("substring")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + final String inputPath = parser.get("sourcePath"); log.info("inputPath: {}", inputPath); @@ -74,11 +81,12 @@ public class SparkPrepareResultProject implements Serializable { isSparkSessionManaged, spark -> { Utils.removeOutputDir(spark, outputPath); - prepareResultProjectList(spark, inputPath, outputPath); + prepareResultProjectList(spark, inputPath, outputPath, substring); }); } - private static void prepareResultProjectList(SparkSession spark, String inputPath, String outputPath) { + private static void prepareResultProjectList(SparkSession spark, String inputPath, String outputPath, + Boolean substring) { Dataset relation = Utils .readPath(spark, inputPath + "/relation", Relation.class) .filter( @@ -101,7 +109,10 @@ public class SparkPrepareResultProject implements Serializable { Set projectSet = new HashSet<>(); Tuple2 first = it.next(); ResultProject rp = new ResultProject(); - rp.setResultId(s); + if (substring) + rp.setResultId(s.substring(3)); + else + rp.setResultId(s); eu.dnetlib.dhp.schema.oaf.Project p = first._1(); projectSet.add(p.getId()); Project ps = getProject(p, first._2); @@ -131,7 +142,7 @@ public class SparkPrepareResultProject implements Serializable { private static Project getProject(eu.dnetlib.dhp.schema.oaf.Project op, Relation relation) { Project p = Project .newInstance( - op.getId(), + op.getId().substring(3), op.getCode().getValue(), Optional .ofNullable(op.getAcronym()) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java index fd2712f..8315808 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java @@ -84,7 +84,7 @@ public class Extractor implements Serializable { .orElse(null)) .orElse(null); Relation r = getRelation( - value.getId().substring(3), contextId.substring(3), + value.getId().substring(3), contextId, Constants.RESULT_ENTITY, Constants.CONTEXT_ENTITY, ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP, provenance); @@ -94,7 +94,7 @@ public class Extractor implements Serializable { hashCodes.add(r.hashCode()); } r = getRelation( - contextId.substring(3), value.getId().substring(3), + contextId, value.getId().substring(3), Constants.CONTEXT_ENTITY, Constants.RESULT_ENTITY, ModelConstants.IS_RELATED_TO, diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java index 55390c6..30fdb96 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java @@ -55,8 +55,8 @@ public class Process implements Serializable { ci .getDatasourceList() .forEach(ds -> { - String nodeType = ModelSupport.idPrefixEntity.get(ds.substring(0, 2)); + String datasourceId = ds.startsWith("10|") || ds.startsWith("40|") ? ds.substring(3) : ds; String contextId = Utils.getContextId(ci.getId()); relationList @@ -65,7 +65,7 @@ public class Process implements Serializable { .newInstance( contextId, eu.dnetlib.dhp.oa.model.graph.Constants.CONTEXT_ENTITY, - ds, nodeType, + datasourceId, nodeType, RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), Provenance .newInstance( @@ -76,7 +76,7 @@ public class Process implements Serializable { .add( Relation .newInstance( - ds, nodeType, + datasourceId, nodeType, contextId, eu.dnetlib.dhp.oa.model.graph.Constants.CONTEXT_ENTITY, RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), Provenance diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java index f6f7c9f..1335c68 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java @@ -105,7 +105,7 @@ public class SparkCollectAndSave implements Serializable { } -// Dataset dumpedIds = Utils.getEntitiesId(spark, outputPath); + Dataset dumpedIds = Utils.getEntitiesId(spark, outputPath); Dataset relations = Utils .readPath(spark, inputPath + "/relation/publication", Relation.class) @@ -116,16 +116,18 @@ public class SparkCollectAndSave implements Serializable { .union(Utils.readPath(spark, inputPath + "/relation/context", Relation.class)) .union(Utils.readPath(spark, inputPath + "/relation/relation", Relation.class)); - Utils.getValidRelations(relations, Utils.getEntitiesId(spark, outputPath)) -// Dataset relJoinSource = relations -// .joinWith(dumpedIds, relations.col("source.id").equalTo(dumpedIds.col("value"))) -// .map((MapFunction, Relation>) t2 -> t2._1(), -// Encoders.bean(Relation.class)); -// -// relJoinSource -// .joinWith(dumpedIds, relJoinSource.col("target.id").equalTo(dumpedIds.col("value"))) -// .map((MapFunction, Relation>) t2 -> t2._1(), -// Encoders.bean(Relation.class)) +// Utils.getValidRelations(relations, Utils.getEntitiesId(spark, outputPath)) + Dataset relJoinSource = relations + .joinWith(dumpedIds, relations.col("source").equalTo(dumpedIds.col("value"))) + .map( + (MapFunction, Relation>) t2 -> t2._1(), + Encoders.bean(Relation.class)); + + relJoinSource + .joinWith(dumpedIds, relJoinSource.col("target").equalTo(dumpedIds.col("value"))) + .map( + (MapFunction, Relation>) t2 -> t2._1(), + Encoders.bean(Relation.class)) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java index 8aeb72d..5c84c55 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java @@ -79,7 +79,9 @@ public class SparkDumpRelationJob implements Serializable { private static void dumpRelation(SparkSession spark, String inputPath, String outputPath, Set removeSet) { Dataset relations = Utils.readPath(spark, inputPath, Relation.class); relations - .filter((FilterFunction) r -> !removeSet.contains(r.getRelClass())) + .filter( + (FilterFunction) r -> !removeSet.contains(r.getRelClass()) + && !r.getSubRelType().equalsIgnoreCase("resultService")) .map((MapFunction) relation -> { eu.dnetlib.dhp.oa.model.graph.Relation relNew = new eu.dnetlib.dhp.oa.model.graph.Relation(); relNew diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java index 473633b..527e324 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java @@ -155,7 +155,7 @@ public class SparkOrganizationRelation implements Serializable { eu.dnetlib.dhp.oa.model.graph.Relation .newInstance( id, Constants.CONTEXT_ENTITY, - organization, + organization.substring(3), ModelSupport.idPrefixEntity.get(organization.substring(0, 2)), RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), Provenance @@ -167,7 +167,7 @@ public class SparkOrganizationRelation implements Serializable { .add( eu.dnetlib.dhp.oa.model.graph.Relation .newInstance( - organization, ModelSupport.idPrefixEntity.get(organization.substring(0, 2)), + organization.substring(3), ModelSupport.idPrefixEntity.get(organization.substring(0, 2)), id, Constants.CONTEXT_ENTITY, RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), Provenance diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/project_prep_parameters.json b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/project_prep_parameters.json index 82714d9..a4ebd34 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/project_prep_parameters.json +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/project_prep_parameters.json @@ -16,5 +16,11 @@ "paramLongName": "isSparkSessionManaged", "paramDescription": "true if the spark session is managed, false otherwise", "paramRequired": false + }, + { + "paramName": "sb", + "paramLongName": "substring", + "paramDescription": "true if the spark session is managed, false otherwise", + "paramRequired": false } ] \ No newline at end of file diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml index 75124cf..d755629 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml @@ -102,6 +102,7 @@ --sourcePath${sourcePath} --outputPath${workingDir}/preparedInfo + --substringfalse From 097905171a8330de82f8349626cccee96e71f9dd Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 4 Aug 2023 16:22:23 +0200 Subject: [PATCH 08/12] adding master duplicate to avoid join of relation. Changed the model for the indicators --- ...actMeasures.java => ImpactIndicators.java} | 2 +- .../eu/dnetlib/dhp/oa/model/Indicator.java | 10 +- .../java/eu/dnetlib/dhp/oa/model/Score.java | 9 + .../eu/dnetlib/dhp/common/MakeTarArchive.java | 190 ++++++++++++++++++ .../eu/dnetlib/dhp/oa/graph/dump/Utils.java | 49 ++--- .../dump/complete/CreateContextRelation.java | 45 ++++- .../dhp/oa/graph/dump/complete/Process.java | 5 +- .../dump/complete/QueryInformationSystem.java | 27 ++- .../dump/complete/SparkCollectAndSave.java | 135 +++++++++++-- .../dump/complete/SparkDumpEntitiesJob.java | 6 + .../SparkSelectValidRelationsJob.java | 58 ++++-- .../dhp/oa/zenodoapi/ZenodoAPIClient.java | 8 +- .../input_maketar_parameters.json | 0 .../oa/graph/dump/input_entity_parameter.json | 7 +- .../graph/dump/wf/main/oozie_app/workflow.xml | 12 ++ .../complete/oozie_app/workflow.xml | 16 +- .../dhp/oa/graph/dump/DumpJobTest.java | 68 +++---- .../dump/complete/CreateRelationTest.java | 153 ++++++++++++-- .../oa/graph/dump/resultDump/dataset_cleaned | 12 +- 19 files changed, 665 insertions(+), 147 deletions(-) rename dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/{ImpactMeasures.java => ImpactIndicators.java} (94%) create mode 100644 dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java rename dump/src/main/resources/eu/dnetlib/dhp/{oa/graph/dump => common}/input_maketar_parameters.json (100%) diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/ImpactMeasures.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/ImpactIndicators.java similarity index 94% rename from dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/ImpactMeasures.java rename to dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/ImpactIndicators.java index 1fdacf1..b98d89e 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/ImpactMeasures.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/ImpactIndicators.java @@ -7,7 +7,7 @@ import java.io.Serializable; * @author miriam.baglioni * @Date 07/11/22 */ -public class ImpactMeasures implements Serializable { +public class ImpactIndicators implements Serializable { Score influence; Score influence_alt; Score popularity; diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Indicator.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Indicator.java index 6459fee..65883f8 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Indicator.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Indicator.java @@ -9,18 +9,18 @@ import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema; public class Indicator implements Serializable { @JsonSchema(description = "The impact measures (i.e. popularity)") - ImpactMeasures impactMeasures; + List bipIndicators; @JsonSchema(description = "The usage counts (i.e. downloads)") UsageCounts usageCounts; @JsonInclude(JsonInclude.Include.NON_NULL) - public ImpactMeasures getImpactMeasures() { - return impactMeasures; + public List getBipIndicators() { + return bipIndicators; } - public void setImpactMeasures(ImpactMeasures impactMeasures) { - this.impactMeasures = impactMeasures; + public void setBipIndicators(List bipIndicators) { + this.bipIndicators = bipIndicators; } @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Score.java b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Score.java index 4b48346..5d0e1dc 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Score.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Score.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.annotation.JsonSetter; * @Date 07/11/22 */ public class Score implements Serializable { + private String indicator; private String score; @JsonProperty("class") @@ -34,4 +35,12 @@ public class Score implements Serializable { public void setClazz(String clazz) { this.clazz = clazz; } + + public String getIndicator() { + return indicator; + } + + public void setIndicator(String indicator) { + this.indicator = indicator; + } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java b/dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java new file mode 100644 index 0000000..43e61e5 --- /dev/null +++ b/dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java @@ -0,0 +1,190 @@ + +package eu.dnetlib.dhp.common; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.util.Optional; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class MakeTarArchive implements Serializable { + + private static final Logger log = LoggerFactory.getLogger(MakeTarArchive.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + MakeTarArchive.class + .getResourceAsStream( + "/eu/dnetlib/dhp/common/input_maketar_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + final String outputPath = parser.get("hdfsPath"); + log.info("hdfsPath: {}", outputPath); + + final String hdfsNameNode = parser.get("nameNode"); + log.info("nameNode: {}", hdfsNameNode); + + final String inputPath = parser.get("sourcePath"); + log.info("input path : {}", inputPath); + + final int gBperSplit = Optional + .ofNullable(parser.get("splitSize")) + .map(Integer::valueOf) + .orElse(10); + + final boolean rename = Optional + .ofNullable(parser.get("rename")) + .map(Boolean::valueOf) + .orElse(Boolean.FALSE); + + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", hdfsNameNode); + + FileSystem fileSystem = FileSystem.get(conf); + + makeTArArchive(fileSystem, inputPath, outputPath, gBperSplit, rename); + + } + + public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit) + throws IOException { + makeTArArchive(fileSystem, inputPath, outputPath, gBperSplit, false); + } + + public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit, + boolean rename) + throws IOException { + + RemoteIterator dirIterator = fileSystem.listLocatedStatus(new Path(inputPath)); + + while (dirIterator.hasNext()) { + LocatedFileStatus fileStatus = dirIterator.next(); + + Path p = fileStatus.getPath(); + String pathString = p.toString(); + String entity = pathString.substring(pathString.lastIndexOf("/") + 1); + + MakeTarArchive.tarMaxSize(fileSystem, pathString, outputPath + "/" + entity, entity, gBperSplit, rename); + } + } + + private static TarArchiveOutputStream getTar(FileSystem fileSystem, String outputPath) throws IOException { + Path hdfsWritePath = new Path(outputPath); + if (fileSystem.exists(hdfsWritePath)) { + fileSystem.delete(hdfsWritePath, true); + + } + return new TarArchiveOutputStream(fileSystem.create(hdfsWritePath).getWrappedStream()); + } + + private static void write(FileSystem fileSystem, String inputPath, String outputPath, String dirName, + boolean rename) + throws IOException { + + Path hdfsWritePath = new Path(outputPath); + if (fileSystem.exists(hdfsWritePath)) { + fileSystem.delete(hdfsWritePath, true); + + } + try (TarArchiveOutputStream ar = new TarArchiveOutputStream( + fileSystem.create(hdfsWritePath).getWrappedStream())) { + + RemoteIterator iterator = fileSystem + .listFiles( + new Path(inputPath), true); + + while (iterator.hasNext()) { + writeCurrentFile(fileSystem, dirName, iterator, ar, 0, rename); + } + + } + } + + public static void tarMaxSize(FileSystem fileSystem, String inputPath, String outputPath, String dir_name, + int gBperSplit, boolean rename) throws IOException { + final long bytesPerSplit = 1024L * 1024L * 1024L * gBperSplit; + + long sourceSize = fileSystem.getContentSummary(new Path(inputPath)).getSpaceConsumed(); + + if (sourceSize < bytesPerSplit) { + write(fileSystem, inputPath, outputPath + ".tar", dir_name, rename); + } else { + int partNum = 0; + + RemoteIterator fileStatusListIterator = fileSystem + .listFiles( + new Path(inputPath), true); + boolean next = fileStatusListIterator.hasNext(); + while (next) { + try (TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + (partNum + 1) + ".tar")) { + + long currentSize = 0; + while (next && currentSize < bytesPerSplit) { + currentSize = writeCurrentFile( + fileSystem, dir_name, fileStatusListIterator, ar, currentSize, rename); + next = fileStatusListIterator.hasNext(); + + } + + partNum += 1; + } + } + } + } + + private static long writeCurrentFile(FileSystem fileSystem, String dirName, + RemoteIterator fileStatusListIterator, + TarArchiveOutputStream ar, long currentSize, boolean rename) throws IOException { + LocatedFileStatus fileStatus = fileStatusListIterator.next(); + + Path p = fileStatus.getPath(); + String pString = p.toString(); + if (!pString.endsWith("_SUCCESS")) { + String name = pString.substring(pString.lastIndexOf("/") + 1); + if (name.startsWith("part-") & name.length() > 10) { + String tmp = name.substring(0, 10); + if (name.contains(".")) { + tmp += name.substring(name.indexOf(".")); + } + name = tmp; + } + if (rename) { + if (name.endsWith(".txt.gz")) + name = name.replace(".txt.gz", ".json.gz"); + } + + TarArchiveEntry entry = new TarArchiveEntry(dirName + "/" + name); + entry.setSize(fileStatus.getLen()); + currentSize += fileStatus.getLen(); + ar.putArchiveEntry(entry); + + InputStream is = fileSystem.open(fileStatus.getPath()); + + BufferedInputStream bis = new BufferedInputStream(is); + + int count; + byte[] data = new byte[1024]; + while ((count = bis.read(data, 0, data.length)) != -1) { + ar.write(data, 0, count); + } + bis.close(); + ar.closeArchiveEntry(); + + } + return currentSize; + } + +} diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java index b3f380e..d3f41d8 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java @@ -6,6 +6,7 @@ import static eu.dnetlib.dhp.oa.graph.dump.Constants.*; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.fs.FileSystem; @@ -13,7 +14,6 @@ import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import org.jetbrains.annotations.NotNull; @@ -23,7 +23,6 @@ import com.google.gson.Gson; import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.graph.dump.complete.Constants; -import eu.dnetlib.dhp.oa.model.ImpactMeasures; import eu.dnetlib.dhp.oa.model.Indicator; import eu.dnetlib.dhp.oa.model.Score; import eu.dnetlib.dhp.oa.model.UsageCounts; @@ -63,7 +62,7 @@ public class Utils { return String .format( - "%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5(id)); } @@ -160,23 +159,26 @@ public class Utils { case USAGE_COUNT_VIEWS: getUsageCounts(i).setViews(m.getUnit().get(0).getValue()); break; - case IMPACT_POPULARITY: - getImpactMeasure(i).setPopularity(getScore(m.getUnit())); - break; - case IMPACT_POPULARITY_ALT: - getImpactMeasure(i).setPopularity_alt(getScore(m.getUnit())); - break; - case IMPACT_IMPULSE: - getImpactMeasure(i).setImpulse(getScore(m.getUnit())); - break; - case IMPACT_INFLUENCE: - getImpactMeasure(i).setInfluence(getScore(m.getUnit())); - break; - case IMPACT_INFLUENCE_ALT: - getImpactMeasure(i).setInfluence_alt(getScore(m.getUnit())); - break; default: + getImpactMeasure(i).add(getScore(m.getId(), m.getUnit())); break; +// case IMPACT_POPULARITY: +// getImpactMeasure(i).setPopularity(getScore(m.getUnit())); +// break; +// case IMPACT_POPULARITY_ALT: +// getImpactMeasure(i).setPopularity_alt(getScore(m.getUnit())); +// break; +// case IMPACT_IMPULSE: +// getImpactMeasure(i).setImpulse(getScore(m.getUnit())); +// break; +// case IMPACT_INFLUENCE: +// getImpactMeasure(i).setInfluence(getScore(m.getUnit())); +// break; +// case IMPACT_INFLUENCE_ALT: +// getImpactMeasure(i).setInfluence_alt(getScore(m.getUnit())); +// break; +// default: +// break; } } @@ -192,15 +194,16 @@ public class Utils { } @NotNull - private static ImpactMeasures getImpactMeasure(Indicator i) { - if (i.getImpactMeasures() == null) { - i.setImpactMeasures(new ImpactMeasures()); + private static List getImpactMeasure(Indicator i) { + if (i.getBipIndicators() == null) { + i.setBipIndicators(new ArrayList<>()); } - return i.getImpactMeasures(); + return i.getBipIndicators(); } - private static Score getScore(List unit) { + private static Score getScore(String indicator, List unit) { Score s = new Score(); + s.setIndicator(indicator); for (KeyValue u : unit) { if (u.getKey().equals("score")) { s.setScore(u.getValue()); diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateContextRelation.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateContextRelation.java index 3706529..0708fc9 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateContextRelation.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateContextRelation.java @@ -1,11 +1,9 @@ package eu.dnetlib.dhp.oa.graph.dump.complete; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Serializable; +import java.io.*; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -14,15 +12,20 @@ import java.util.function.Function; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.graph.dump.exceptions.MyRuntimeException; +import eu.dnetlib.dhp.oa.graph.dump.subset.MasterDuplicate; +import eu.dnetlib.dhp.oa.graph.dump.subset.ReadMasterDuplicateFromDB; import eu.dnetlib.dhp.oa.model.graph.*; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.Datasource; @@ -63,15 +66,23 @@ public class CreateContextRelation implements Serializable { log.info("hdfsPath: {}", hdfsPath); final String hdfsNameNode = parser.get("nameNode"); - log.info("nameNode: {}", hdfsNameNode); + log.info("hdfsNameNode: {}", hdfsNameNode); final String isLookUpUrl = parser.get("isLookUpUrl"); log.info("isLookUpUrl: {}", isLookUpUrl); + final String masterDuplicatePath = parser.get("masterDuplicate"); + log.info("masterDuplicatePath: {}", masterDuplicatePath); + final CreateContextRelation cce = new CreateContextRelation(hdfsPath, hdfsNameNode, isLookUpUrl); + final List masterDuplicateList = cce.readMasterDuplicate(masterDuplicatePath); + log.info("Creating relation for datasource..."); - cce.execute(Process::getRelation, CONTEX_RELATION_DATASOURCE, ModelSupport.getIdPrefix(Datasource.class)); + cce + .execute( + Process::getRelation, CONTEX_RELATION_DATASOURCE, ModelSupport.getIdPrefix(Datasource.class), + masterDuplicateList); log.info("Creating relations for projects... "); cce @@ -83,6 +94,20 @@ public class CreateContextRelation implements Serializable { } + private List readMasterDuplicate(String masterDuplicatePath) throws IOException { + FileSystem fileSystem = FileSystem.get(conf); + Path hdfsReadPath = new Path(masterDuplicatePath); + BufferedReader reader = new BufferedReader(new InputStreamReader(fileSystem.open(hdfsReadPath))); + List mdlist = new ArrayList<>(); + ObjectMapper mapper = new ObjectMapper(); + String line; + + while ((line = reader.readLine()) != null) { + mdlist.add(mapper.readValue(line, MasterDuplicate.class)); + } + return mdlist; + } + private void close() throws IOException { writer.close(); } @@ -111,9 +136,15 @@ public class CreateContextRelation implements Serializable { public void execute(final Function> producer, String category, String prefix) { + execute(producer, category, prefix, null); + } + + public void execute(final Function> producer, String category, String prefix, + List masterDuplicateList) { + final Consumer consumer = ci -> producer.apply(ci).forEach(this::writeEntity); - queryInformationSystem.getContextRelation(consumer, category, prefix); + queryInformationSystem.getContextRelation(consumer, category, prefix, masterDuplicateList); } protected void writeEntity(final Relation r) { diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java index 30fdb96..9a46d05 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java @@ -56,14 +56,15 @@ public class Process implements Serializable { .getDatasourceList() .forEach(ds -> { String nodeType = ModelSupport.idPrefixEntity.get(ds.substring(0, 2)); - String datasourceId = ds.startsWith("10|") || ds.startsWith("40|") ? ds.substring(3) : ds; + String datasourceId = ds; + if (ds.startsWith("10|") || ds.startsWith("40|")) + datasourceId = ds.substring(3); String contextId = Utils.getContextId(ci.getId()); relationList .add( Relation .newInstance( - contextId, eu.dnetlib.dhp.oa.model.graph.Constants.CONTEXT_ENTITY, datasourceId, nodeType, RelType.newInstance(ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java index bbcb1a2..d78370a 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java @@ -18,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; +import eu.dnetlib.dhp.oa.graph.dump.subset.MasterDuplicate; import eu.dnetlib.dhp.oa.graph.dump.subset.SparkDumpResult; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.utils.DHPUtils; @@ -110,6 +111,11 @@ public class QueryInformationSystem { } public void getContextRelation(final Consumer consumer, String category, String prefix) { + getContextRelation(consumer, category, prefix, null); + } + + public void getContextRelation(final Consumer consumer, String category, String prefix, + List masterDuplicateList) { contextRelationResult.forEach(xml -> { ContextInfo cinfo = new ContextInfo(); @@ -129,7 +135,7 @@ public class QueryInformationSystem { String categoryId = el.attributeValue("id"); categoryId = categoryId.substring(categoryId.lastIndexOf("::") + 2); if (categoryId.equals(category)) { - cinfo.setDatasourceList(getCategoryList(el, prefix)); + cinfo.setDatasourceList(getCategoryList(el, prefix, masterDuplicateList)); } } @@ -144,17 +150,28 @@ public class QueryInformationSystem { } @NotNull - private List getCategoryList(Element el, String prefix) { + private List getCategoryList(Element el, String prefix, List masterDuplicateList) { List datasourceList = new ArrayList<>(); for (Object node : el.selectNodes(".//concept")) { String oid = getOpenaireId((Node) node, prefix); if (oid != null) - datasourceList.add(oid); + if (masterDuplicateList == null) + datasourceList.add(oid); + else + datasourceList.add(getMaster(oid, masterDuplicateList)); } return datasourceList; } + private String getMaster(String oid, List masterDuplicateList) { + for (MasterDuplicate md : masterDuplicateList) { + if (md.getDuplicate().equals(oid)) + return md.getMaster(); + } + return oid; + } + private String getOpenaireId(Node el, String prefix) { for (Object node : el.selectNodes(".//param")) { Node n = (Node) node; @@ -199,7 +216,9 @@ public class QueryInformationSystem { } if (funding.toLowerCase().contains("h2020")) { nsp = "corda__h2020::"; - } else { + } else if (funding.toLowerCase().contains("he")){ + nsp = "corda_____he::"; + }else{ nsp = "corda_______::"; } break; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java index 1335c68..ab37b71 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkCollectAndSave.java @@ -105,34 +105,139 @@ public class SparkCollectAndSave implements Serializable { } - Dataset dumpedIds = Utils.getEntitiesId(spark, outputPath); + // Dataset dumpedIds = Utils.getEntitiesId(spark, outputPath); - Dataset relations = Utils - .readPath(spark, inputPath + "/relation/publication", Relation.class) - .union(Utils.readPath(spark, inputPath + "/relation/dataset", Relation.class)) - .union(Utils.readPath(spark, inputPath + "/relation/orp", Relation.class)) - .union(Utils.readPath(spark, inputPath + "/relation/software", Relation.class)) - .union(Utils.readPath(spark, inputPath + "/relation/contextOrg", Relation.class)) - .union(Utils.readPath(spark, inputPath + "/relation/context", Relation.class)) - .union(Utils.readPath(spark, inputPath + "/relation/relation", Relation.class)); +// Dataset relations = Utils +// .readPath(spark, inputPath + "/relation/publication", Relation.class) +// .union(Utils.readPath(spark, inputPath + "/relation/dataset", Relation.class)) +// .union(Utils.readPath(spark, inputPath + "/relation/orp", Relation.class)) +// .union(Utils.readPath(spark, inputPath + "/relation/software", Relation.class)) +// .union(Utils.readPath(spark, inputPath + "/relation/contextOrg", Relation.class)) +// .union(Utils.readPath(spark, inputPath + "/relation/context", Relation.class)) +// .union(Utils.readPath(spark, inputPath + "/relation/relation", Relation.class)); // Utils.getValidRelations(relations, Utils.getEntitiesId(spark, outputPath)) - Dataset relJoinSource = relations - .joinWith(dumpedIds, relations.col("source").equalTo(dumpedIds.col("value"))) - .map( - (MapFunction, Relation>) t2 -> t2._1(), - Encoders.bean(Relation.class)); + Utils + .readPath(spark, inputPath + "/relation/publication", Relation.class) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath + "/relation"); + Utils + .readPath(spark, inputPath + "/relation/dataset", Relation.class) + .write() + .mode(SaveMode.Append) + .option("compression", "gzip") + .json(outputPath + "/relation"); + Utils + .readPath(spark, inputPath + "/relation/orp", Relation.class) + .write() + .mode(SaveMode.Append) + .option("compression", "gzip") + .json(outputPath + "/relation"); + Utils + .readPath(spark, inputPath + "/relation/software", Relation.class) + .write() + .mode(SaveMode.Append) + .option("compression", "gzip") + .json(outputPath + "/relation"); + Utils + .readPath(spark, inputPath + "/relation/contextOrg", Relation.class) + .write() + .mode(SaveMode.Append) + .option("compression", "gzip") + .json(outputPath + "/relation"); + Utils + .readPath(spark, inputPath + "/relation/context", Relation.class) + .write() + .mode(SaveMode.Append) + .option("compression", "gzip") + .json(outputPath + "/relation"); + Utils + .readPath(spark, inputPath + "/relation/relation", Relation.class) + .write() + .mode(SaveMode.Append) + .option("compression", "gzip") + .json(outputPath + "/relation"); +// relSource( +// inputPath, dumpedIds, Utils +// .readPath(spark, inputPath + "/relation/publication", Relation.class), +// inputPath + "/relSource/publication"); +// relSource( +// inputPath, dumpedIds, Utils +// .readPath(spark, inputPath + "/relation/dataset", Relation.class), +// inputPath + "/relSource/dataset"); +// relSource( +// inputPath, dumpedIds, Utils +// .readPath(spark, inputPath + "/relation/orp", Relation.class), +// inputPath + "/relSource/orp"); +// relSource( +// inputPath, dumpedIds, Utils +// .readPath(spark, inputPath + "/relation/software", Relation.class), +// inputPath + "/relSource/software"); +// relSource( +// inputPath, dumpedIds, Utils +// .readPath(spark, inputPath + "/relation/contextOrg", Relation.class), +// inputPath + "/relSource/contextOrg"); +// relSource( +// inputPath, dumpedIds, Utils +// .readPath(spark, inputPath + "/relation/context", Relation.class), +// inputPath + "/relSource/context"); +// relSource( +// inputPath, dumpedIds, Utils +// .readPath(spark, inputPath + "/relation/relation", Relation.class), +// inputPath + "/relSource/relation"); + +// relTarget( +// outputPath, dumpedIds, Utils.readPath(spark, inputPath + "/relSource/publication", Relation.class), +// SaveMode.Overwrite); +// relTarget( +// outputPath, dumpedIds, Utils.readPath(spark, inputPath + "/relSource/dataset", Relation.class), +// SaveMode.Append); +// relTarget( +// outputPath, dumpedIds, Utils.readPath(spark, inputPath + "/relSource/orp", Relation.class), +// SaveMode.Append); +// relTarget( +// outputPath, dumpedIds, Utils.readPath(spark, inputPath + "/relSource/software", Relation.class), +// SaveMode.Append); +// relTarget( +// outputPath, dumpedIds, Utils.readPath(spark, inputPath + "/relSource/contextOrg", Relation.class), +// SaveMode.Append); +// relTarget( +// outputPath, dumpedIds, Utils.readPath(spark, inputPath + "/relSource/context", Relation.class), +// SaveMode.Append); +// relTarget( +// outputPath, dumpedIds, Utils.readPath(spark, inputPath + "/relSource/relation", Relation.class), +// SaveMode.Append); + + } + + private static void relTarget(String outputPath, Dataset dumpedIds, Dataset relJoinSource, + SaveMode saveMode) { relJoinSource .joinWith(dumpedIds, relJoinSource.col("target").equalTo(dumpedIds.col("value"))) .map( (MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)) .write() - .mode(SaveMode.Overwrite) + .mode(saveMode) .option("compression", "gzip") .json(outputPath + "/relation"); + } + private static void relSource(String inputPath, Dataset dumpedIds, Dataset relations, + String outputPath) { + + relations + .joinWith(dumpedIds, relations.col("source").equalTo(dumpedIds.col("value"))) + .map( + (MapFunction, Relation>) t2 -> t2._1(), + Encoders.bean(Relation.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); } private static void write(Dataset dataSet, String outputPath) { diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java index 2ac7bf4..2d80d67 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java @@ -204,6 +204,7 @@ public class SparkDumpEntitiesJob implements Serializable { .map( (MapFunction) p -> mapProject((eu.dnetlib.dhp.schema.oaf.Project) p), Encoders.bean(Project.class)) + .filter((FilterFunction) p -> p != null) .write() .mode(SaveMode.Overwrite) .option(COMPRESSION, GZIP) @@ -211,6 +212,8 @@ public class SparkDumpEntitiesJob implements Serializable { } private static Datasource mapDatasource(eu.dnetlib.dhp.schema.oaf.Datasource d) { + if (Boolean.TRUE.equals(d.getDataInfo().getDeletedbyinference())) + return null; Datasource datasource = new Datasource(); datasource.setId(d.getId().substring(3)); @@ -396,6 +399,9 @@ public class SparkDumpEntitiesJob implements Serializable { } private static Project mapProject(eu.dnetlib.dhp.schema.oaf.Project p) throws DocumentException { + if (Boolean.TRUE.equals(p.getDataInfo().getDeletedbyinference())) + return null; + Project project = new Project(); Optional diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java index 1e5675e..73ebc21 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java @@ -8,6 +8,7 @@ import java.util.Optional; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SaveMode; @@ -66,16 +67,45 @@ public class SparkSelectValidRelationsJob implements Serializable { } private static void selectValidRelation(SparkSession spark, String inputPath, String outputPath) { - Dataset relation = Utils.readPath(spark, inputPath + "/relation", Relation.class); - Dataset publication = Utils.readPath(spark, inputPath + "/publication", Publication.class); + Dataset relation = Utils + .readPath(spark, inputPath + "/relation", Relation.class) + .filter((FilterFunction) r -> !r.getDataInfo().getDeletedbyinference()); + Dataset publication = Utils + .readPath(spark, inputPath + "/publication", Publication.class) + .filter( + (FilterFunction) p -> !p.getDataInfo().getDeletedbyinference() + && !p.getDataInfo().getInvisible()); Dataset dataset = Utils - .readPath(spark, inputPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class); - Dataset software = Utils.readPath(spark, inputPath + "/software", Software.class); + .readPath(spark, inputPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class) + .filter( + (FilterFunction) d -> !d.getDataInfo().getDeletedbyinference() + && !d.getDataInfo().getInvisible()); + ; + Dataset software = Utils + .readPath(spark, inputPath + "/software", Software.class) + .filter( + (FilterFunction) s -> !s.getDataInfo().getDeletedbyinference() + && !s.getDataInfo().getInvisible()); Dataset other = Utils - .readPath(spark, inputPath + "/otherresearchproduct", OtherResearchProduct.class); - Dataset organization = Utils.readPath(spark, inputPath + "/organization", Organization.class); - Dataset project = Utils.readPath(spark, inputPath + "/project", Project.class); - Dataset datasource = Utils.readPath(spark, inputPath + "/datasource", Datasource.class); + .readPath(spark, inputPath + "/otherresearchproduct", OtherResearchProduct.class) + .filter( + (FilterFunction) o -> !o.getDataInfo().getDeletedbyinference() + && !o.getDataInfo().getInvisible()); + Dataset organization = Utils + .readPath(spark, inputPath + "/organization", Organization.class) + .filter( + (FilterFunction) o -> !o.getDataInfo().getDeletedbyinference() + && !o.getDataInfo().getInvisible()); + Dataset project = Utils + .readPath(spark, inputPath + "/project", Project.class) + .filter( + (FilterFunction) p -> !p.getDataInfo().getDeletedbyinference() + && !p.getDataInfo().getInvisible()); + Dataset datasource = Utils + .readPath(spark, inputPath + "/datasource", Datasource.class) + .filter( + (FilterFunction) d -> !d.getDataInfo().getDeletedbyinference() + && !d.getDataInfo().getInvisible()); relation.createOrReplaceTempView("relation"); publication.createOrReplaceTempView("publication"); @@ -90,31 +120,24 @@ public class SparkSelectValidRelationsJob implements Serializable { .sql( "SELECT id " + "FROM publication " + - "WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " + "UNION ALL " + "SELECT id " + "FROM dataset " + - "WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " + "UNION ALL " + "SELECT id " + "FROM other " + - "WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " + "UNION ALL " + "SELECT id " + "FROM software " + - "WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " + "UNION ALL " + "SELECT id " + "FROM organization " + - "WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " + "UNION ALL " + "SELECT id " + "FROM project " + - "WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " + "UNION ALL " + "SELECT id " + - "FROM datasource " + - "WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false ") + "FROM datasource ") .createOrReplaceTempView("identifiers"); spark @@ -124,8 +147,7 @@ public class SparkSelectValidRelationsJob implements Serializable { "JOIN identifiers i1 " + "ON source = i1.id " + "JOIN identifiers i2 " + - "ON target = i2.id " + - "WHERE datainfo.deletedbyinference = false") + "ON target = i2.id ") .as(Encoders.bean(Relation.class)) .write() .option("compression", "gzip") diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java index d37be74..59f6055 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java @@ -122,8 +122,8 @@ public class ZenodoAPIClient implements Serializable { // Get response body json = response.body().string(); - eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson() - .fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class); + ZenodoModel newSubmission = new Gson() + .fromJson(json, ZenodoModel.class); this.bucket = newSubmission.getLinks().getBucket(); this.deposition_id = newSubmission.getId(); @@ -386,8 +386,8 @@ public class ZenodoAPIClient implements Serializable { if (!response.isSuccessful()) throw new IOException("Unexpected code " + response + response.body().string()); - eu.dnetlib.dhp.common.api.zenodo.ZenodoModel zenodoModel = new Gson() - .fromJson(response.body().string(), eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class); + ZenodoModel zenodoModel = new Gson() + .fromJson(response.body().string(), ZenodoModel.class); bucket = zenodoModel.getLinks().getBucket(); return response.code(); diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_maketar_parameters.json b/dump/src/main/resources/eu/dnetlib/dhp/common/input_maketar_parameters.json similarity index 100% rename from dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_maketar_parameters.json rename to dump/src/main/resources/eu/dnetlib/dhp/common/input_maketar_parameters.json diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_entity_parameter.json b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_entity_parameter.json index 87de13d..9946e94 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_entity_parameter.json +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_entity_parameter.json @@ -17,7 +17,12 @@ "paramLongName": "nameNode", "paramDescription": "the name node", "paramRequired": true - } + },{ + "paramName": "md", + "paramLongName": "masterDuplicate", + "paramDescription": "the master duplicate path for datasource deduplication", + "paramRequired": false +} ] diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/main/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/main/oozie_app/workflow.xml index ffa8bcd..32226fc 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/main/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/main/oozie_app/workflow.xml @@ -256,6 +256,18 @@ resultAggregation ${resultAggregation} + + postgresURL + ${postgresURL} + + + postgresUser + ${postgresUser} + + + postgresPassword + ${postgresPassword} + diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/complete/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/complete/oozie_app/workflow.xml index ab598f4..4b9983b 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/complete/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/complete/oozie_app/workflow.xml @@ -85,9 +85,20 @@ - - + + + + eu.dnetlib.dhp.oa.graph.dump.subset.ReadMasterDuplicateFromDB + --hdfsPath${workingDir}/masterduplicate + --hdfsNameNode${nameNode} + --postgresUrl${postgresURL} + --postgresUser${postgresUser} + --postgresPassword${postgresPassword} + + + + @@ -350,6 +361,7 @@ --hdfsPath${workingDir}/relation/context --nameNode${nameNode} --isLookUpUrl${isLookUpUrl} + --masterDuplicate${workingDir}/masterduplicate diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java index 9f5c05b..2496219 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.graph.dump; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Optional; import org.apache.commons.io.FileUtils; @@ -20,26 +21,16 @@ import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.graph.dump.community.SparkDumpCommunityProducts; import eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpEntitiesJob; -import eu.dnetlib.dhp.oa.graph.dump.subset.SparkDumpResult; -import eu.dnetlib.dhp.oa.graph.dump.subset.criteria.VerbResolver; -import eu.dnetlib.dhp.oa.graph.dump.subset.criteria.VerbResolverFactory; -import eu.dnetlib.dhp.oa.graph.dump.subset.selectionconstraints.ProtoMap; -import eu.dnetlib.dhp.oa.graph.dump.subset.selectionconstraints.SelectionConstraints; import eu.dnetlib.dhp.oa.model.*; import eu.dnetlib.dhp.oa.model.community.CommunityResult; import eu.dnetlib.dhp.oa.model.graph.GraphResult; import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.Dataset; -import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct; -import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.dhp.schema.oaf.Software; //@Disabled public class DumpJobTest { @@ -149,7 +140,7 @@ public class DumpJobTest { GraphResult gr = verificationDataset.first(); Assertions.assertTrue(Optional.ofNullable(gr.getIndicators().getUsageCounts()).isPresent()); - Assertions.assertFalse(Optional.ofNullable(gr.getIndicators().getImpactMeasures()).isPresent()); + Assertions.assertFalse(Optional.ofNullable(gr.getIndicators().getBipIndicators()).isPresent()); } @@ -372,19 +363,11 @@ public class DumpJobTest { Assertions.assertEquals("2020-03-23T00:20:51.392Z", gr.getDateofcollection()); Assertions.assertTrue(Optional.ofNullable(gr.getIndicators().getUsageCounts()).isPresent()); - Assertions.assertTrue(Optional.ofNullable(gr.getIndicators().getImpactMeasures()).isPresent()); + Assertions.assertTrue(Optional.ofNullable(gr.getIndicators().getBipIndicators()).isPresent()); Assertions - .assertTrue(gr.getIndicators().getImpactMeasures().getInfluence() != null); - Assertions - .assertTrue(gr.getIndicators().getImpactMeasures().getPopularity_alt() != null); + .assertEquals(5, gr.getIndicators().getBipIndicators().size()); - Assertions - .assertTrue(gr.getIndicators().getImpactMeasures().getPopularity() != null); - Assertions - .assertTrue(gr.getIndicators().getImpactMeasures().getInfluence_alt() != null); - Assertions - .assertTrue(gr.getIndicators().getImpactMeasures().getImpulse() != null); Assertions .assertTrue(gr.getIndicators().getUsageCounts() != null); Assertions @@ -392,20 +375,31 @@ public class DumpJobTest { Assertions .assertTrue(Integer.valueOf(gr.getIndicators().getUsageCounts().getViews()) >= 0); - Assertions.assertEquals("6.01504990349e-09", gr.getIndicators().getImpactMeasures().getInfluence().getScore()); - Assertions.assertEquals("C", gr.getIndicators().getImpactMeasures().getInfluence().getClazz()); + List bip = gr.getIndicators().getBipIndicators(); + for (Score in : bip) { + switch (in.getIndicator()) { + case "influence": + Assertions.assertEquals("6.01504990349e-09", in.getScore()); + Assertions.assertEquals("C", in.getClazz()); + break; + case "popularity_alt": + Assertions.assertEquals("2.304", in.getScore()); + Assertions.assertEquals("C", in.getClazz()); + break; + case "popularity": + Assertions.assertEquals("1.81666032463e-08", in.getScore()); + Assertions.assertEquals("C", in.getClazz()); + break; + case "influence_alt": + Assertions.assertEquals("8.0", in.getScore()); + Assertions.assertEquals("C", in.getClazz()); + break; + case "impulse": + Assertions.assertEquals("8.0", in.getScore()); + Assertions.assertEquals("C", in.getClazz()); + } - Assertions.assertEquals("2.304", gr.getIndicators().getImpactMeasures().getPopularity_alt().getScore()); - Assertions.assertEquals("C", gr.getIndicators().getImpactMeasures().getPopularity_alt().getClazz()); - - Assertions.assertEquals("1.81666032463e-08", gr.getIndicators().getImpactMeasures().getPopularity().getScore()); - Assertions.assertEquals("C", gr.getIndicators().getImpactMeasures().getPopularity().getClazz()); - - Assertions.assertEquals("8.0", gr.getIndicators().getImpactMeasures().getInfluence_alt().getScore()); - Assertions.assertEquals("C", gr.getIndicators().getImpactMeasures().getInfluence_alt().getClazz()); - - Assertions.assertEquals("8.0", gr.getIndicators().getImpactMeasures().getImpulse().getScore()); - Assertions.assertEquals("C", gr.getIndicators().getImpactMeasures().getImpulse().getClazz()); + } Assertions.assertEquals("0", gr.getIndicators().getUsageCounts().getDownloads()); Assertions.assertEquals("1", gr.getIndicators().getUsageCounts().getViews()); @@ -686,18 +680,18 @@ public class DumpJobTest { Assertions.assertEquals(1, cr.getCollectedfrom().size()); Assertions - .assertEquals("10|openaire____::fdc7e0400d8c1634cdaf8051dbae23db", cr.getCollectedfrom().get(0).getKey()); + .assertEquals("openaire____::fdc7e0400d8c1634cdaf8051dbae23db", cr.getCollectedfrom().get(0).getKey()); Assertions.assertEquals("Pensoft", cr.getCollectedfrom().get(0).getValue()); Assertions.assertEquals(1, cr.getInstance().size()); Assertions .assertEquals( - "10|openaire____::fdc7e0400d8c1634cdaf8051dbae23db", + "openaire____::fdc7e0400d8c1634cdaf8051dbae23db", cr.getInstance().get(0).getCollectedfrom().getKey()); Assertions.assertEquals("Pensoft", cr.getInstance().get(0).getCollectedfrom().getValue()); Assertions .assertEquals( - "10|openaire____::e707e544b9a5bd23fc27fbfa65eb60dd", cr.getInstance().get(0).getHostedby().getKey()); + "openaire____::e707e544b9a5bd23fc27fbfa65eb60dd", cr.getInstance().get(0).getHostedby().getKey()); Assertions.assertEquals("One Ecosystem", cr.getInstance().get(0).getHostedby().getValue()); } diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateRelationTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateRelationTest.java index e01cb3e..9b1fb7a 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateRelationTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateRelationTest.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import eu.dnetlib.dhp.oa.graph.dump.subset.MasterDuplicate; import eu.dnetlib.dhp.oa.model.graph.Relation; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.Datasource; @@ -567,7 +568,7 @@ class CreateRelationTest { .contains( String .format( - "%s|%s::%s", Constants.CONTEXT_ID, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("dh-ch")))); @@ -583,7 +584,7 @@ class CreateRelationTest { .equals( String .format( - "%s|%s::%s", Constants.CONTEXT_ID, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("dh-ch")))) .collect(Collectors.toList()) @@ -601,7 +602,7 @@ class CreateRelationTest { .equals( String .format( - "%s|%s::%s", Constants.CONTEXT_ID, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("dh-ch")))) .collect(Collectors.toList()) @@ -616,7 +617,7 @@ class CreateRelationTest { .equals( String .format( - "%s|%s::%s", Constants.CONTEXT_ID, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("dh-ch")))) .map(r -> r.getTarget()) @@ -624,16 +625,16 @@ class CreateRelationTest { Assertions .assertTrue( - tmp.contains("10|re3data_____::9ebe127e5f3a0bf401875690f3bb6b81") && - tmp.contains("10|doajarticles::c6cd4b532e12868c1d760a8d7cda6815") && - tmp.contains("10|doajarticles::a6de4499bb87bf3c01add0a9e2c9ed0b") && - tmp.contains("10|doajarticles::6eb31d13b12bc06bbac06aef63cf33c9") && - tmp.contains("10|doajarticles::0da84e9dfdc8419576169e027baa8028") && - tmp.contains("10|re3data_____::84e123776089ce3c7a33db98d9cd15a8") && - tmp.contains("10|openaire____::c5502a43e76feab55dd00cf50f519125") && - tmp.contains("10|re3data_____::a48f09c562b247a9919acfe195549b47") && - tmp.contains("10|opendoar____::97275a23ca44226c9964043c8462be96") && - tmp.contains("10|doajarticles::2899208a99aa7d142646e0a80bfeef05")); + tmp.contains("re3data_____::9ebe127e5f3a0bf401875690f3bb6b81") && + tmp.contains("doajarticles::c6cd4b532e12868c1d760a8d7cda6815") && + tmp.contains("doajarticles::a6de4499bb87bf3c01add0a9e2c9ed0b") && + tmp.contains("doajarticles::6eb31d13b12bc06bbac06aef63cf33c9") && + tmp.contains("doajarticles::0da84e9dfdc8419576169e027baa8028") && + tmp.contains("re3data_____::84e123776089ce3c7a33db98d9cd15a8") && + tmp.contains("openaire____::c5502a43e76feab55dd00cf50f519125") && + tmp.contains("re3data_____::a48f09c562b247a9919acfe195549b47") && + tmp.contains("opendoar____::97275a23ca44226c9964043c8462be96") && + tmp.contains("doajarticles::2899208a99aa7d142646e0a80bfeef05")); } @@ -662,7 +663,7 @@ class CreateRelationTest { .contains( String .format( - "%s|%s::%s", Constants.CONTEXT_ID, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("dh-ch")))); @@ -678,7 +679,7 @@ class CreateRelationTest { .equals( String .format( - "%s|%s::%s", Constants.CONTEXT_ID, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("clarin")))) .collect(Collectors.toList()) @@ -696,7 +697,7 @@ class CreateRelationTest { .equals( String .format( - "%s|%s::%s", Constants.CONTEXT_ID, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("clarin")))) .collect(Collectors.toList()) @@ -711,7 +712,7 @@ class CreateRelationTest { .equals( String .format( - "%s|%s::%s", Constants.CONTEXT_ID, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5("clarin")))) .map(r -> r.getTarget()) @@ -719,15 +720,123 @@ class CreateRelationTest { Assertions .assertTrue( - tmp.contains("40|corda__h2020::b5a4eb56bf84bef2ebc193306b4d423f") && - tmp.contains("40|corda_______::ef782b2d85676aa3e5a907427feb18c4")); + tmp.contains("corda__h2020::b5a4eb56bf84bef2ebc193306b4d423f") && + tmp.contains("corda_______::ef782b2d85676aa3e5a907427feb18c4")); rList.forEach(rel -> { - if (rel.getSource().startsWith("40|")) { - String proj = rel.getSource().substring(3); + if (rel.getSourceType().equals("project")) { + String proj = rel.getSource(); Assertions.assertTrue(proj.substring(0, proj.indexOf("::")).length() == 12); + Assertions.assertFalse(proj.startsWith("40|")); } }); } + + @Test + void test3() { + List cInfoList = new ArrayList<>(); + final Consumer consumer = ci -> cInfoList.add(ci); + + MasterDuplicate md1 = new MasterDuplicate(); + md1.setMaster("10|fake________::9ebe127e5f3a0bf401875690f3bb6b81"); + md1.setDuplicate("10|re3data_____::9ebe127e5f3a0bf401875690f3bb6b81"); + queryInformationSystem + .getContextRelation( + consumer, "contentproviders", ModelSupport.getIdPrefix(Datasource.class), Arrays.asList(md1)); + + cInfoList.forEach(c -> System.out.println(new Gson().toJson(c))); + + List rList = new ArrayList<>(); + + cInfoList.forEach(cInfo -> Process.getRelation(cInfo).forEach(rList::add)); + + rList.forEach(r -> { + try { + System.out.println(new ObjectMapper().writeValueAsString(r)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + }); + + Assertions.assertEquals(34, rList.size()); + + Assertions + .assertTrue( + rList + .stream() + .map(r -> r.getSource()) + .collect(Collectors.toSet()) + .contains( + String + .format( + "%s::%s", + Constants.CONTEXT_NS_PREFIX, + DHPUtils.md5("dh-ch")))); + + Assertions + .assertEquals( + 10, + rList + .stream() + .filter( + r -> r + .getSource() + + .equals( + String + .format( + "%s::%s", + Constants.CONTEXT_NS_PREFIX, + DHPUtils.md5("dh-ch")))) + .collect(Collectors.toList()) + .size()); + + Assertions + .assertEquals( + 10, + rList + .stream() + .filter( + r -> r + .getTarget() + + .equals( + String + .format( + "%s::%s", + Constants.CONTEXT_NS_PREFIX, + DHPUtils.md5("dh-ch")))) + .collect(Collectors.toList()) + .size()); + + Set tmp = rList + .stream() + .filter( + r -> r + .getSource() + + .equals( + String + .format( + "%s::%s", + Constants.CONTEXT_NS_PREFIX, + DHPUtils.md5("dh-ch")))) + .map(r -> r.getTarget()) + .collect(Collectors.toSet()); + + Assertions + .assertTrue( + tmp.contains("fake________::9ebe127e5f3a0bf401875690f3bb6b81") && + tmp.contains("doajarticles::c6cd4b532e12868c1d760a8d7cda6815") && + tmp.contains("doajarticles::a6de4499bb87bf3c01add0a9e2c9ed0b") && + tmp.contains("doajarticles::6eb31d13b12bc06bbac06aef63cf33c9") && + tmp.contains("doajarticles::0da84e9dfdc8419576169e027baa8028") && + tmp.contains("re3data_____::84e123776089ce3c7a33db98d9cd15a8") && + tmp.contains("openaire____::c5502a43e76feab55dd00cf50f519125") && + tmp.contains("re3data_____::a48f09c562b247a9919acfe195549b47") && + tmp.contains("opendoar____::97275a23ca44226c9964043c8462be96") && + tmp.contains("doajarticles::2899208a99aa7d142646e0a80bfeef05")); + + } } diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_cleaned b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_cleaned index 2b8ab40..dc920d6 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_cleaned +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_cleaned @@ -1,7 +1,7 @@ {"author":[{"fullname":"Yan Hao Zhang","name":"Yan Hao","rank":1,"surname":"Zhang"},{"fullname":"Chao Li","name":"Chao","rank":2,"surname":"Li"},{"fullname":"Jian Sheng Zhao","name":"Jian Sheng","rank":3,"surname":"Zhao"},{"fullname":"Wen Tao Li","name":"Wen Tao","rank":4,"surname":"Li"},{"fullname":"Pei Dong Zhang","name":"Pei Dong","rank":5,"surname":"Zhang"}],"bestaccessright":{"classid":"RESTRICTED","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:actionset","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"value":"2019-12-03T23:04:25Z"},"dateofcollection":"2019-12-03T23:40:54Z","description":[],"id":"60|doiboost____::4eb63cb600dfa04a33ac338e6a72cd0f","instance":[{"accessright":{"classid":"RESTRICTED","classname":"Restricted","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"},"hostedby":{"key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.22541/au.157541426.63709126"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1575416454963,"originalId":["10.22541/au.157541426.63709126"],"pid":[{"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.22541/au.157541426.63709126"}],"publisher":{"value":"Authorea, Inc."},"relevantdate":[{"qualifier":{"classid":"created","classname":"created","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"value":"2019-12-03T23:04:25Z"}],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[{"value":"Crossref"}],"title":[{"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Seagrass resilience: where and how to collect donor plants for ecological restoration of eelgrass Zostera marina"}]} -{"author":[{"affiliation":[],"fullname":"Daniels, Matthew J.","name":"Matthew J.","pid":[],"rank":1,"surname":"Daniels"},{"affiliation":[],"fullname":"Newton, James D.","name":"James D.","pid":[],"rank":2,"surname":"Newton"},{"affiliation":[],"fullname":"Kelion, Andrew D.","name":"Andrew D.","pid":[],"rank":3,"surname":"Kelion"},{"affiliation":[],"fullname":"Petrou, Mario","name":"Mario","pid":[],"rank":4,"surname":"Petrou"},{"affiliation":[],"fullname":"Ormerod, Oliver J.","name":"Oliver J.","pid":[],"rank":5,"surname":"Ormerod"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|re3data_____::7980778c78fb4cf0fab13ce2159030dc","value":"figshare"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2018-01-01"},"dateofcollection":"","dateoftransformation":"2020-06-19T22:37:45.636Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Percutaneous Management of Acquired Right Ventricular Outflow Tract Obstruction due to Giant Coronary Vein Graft Aneurysm"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|r37980778c78::cddbf68617b0ae86a263f25b01d1d06d","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|re3data_____::7980778c78fb4cf0fab13ce2159030dc","value":"figshare"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2018-01-01"},"distributionlocation":"","hostedby":{"key":"10|re3data_____::7980778c78fb4cf0fab13ce2159030dc","value":"figshare"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.6084/m9.figshare.7221605.v2","https://figshare.com/articles/Percutaneous_management_of_acquired_right_ventricular_outflow_tract_obstruction_due_to_giant_coronary_vein_graft_aneurysm/7221605"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594988586710,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"https%3A%2F%2Fapi.figshare.com%2Fv2%2Foai","datestamp":"2018-10-17T21:29:18Z","harvestDate":"2020-06-19T13:26:40.229Z","identifier":"oai:figshare.com:article/7221605","metadataNamespace":""}},"originalId":["r37980778c78::cddbf68617b0ae86a263f25b01d1d06d"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.6084/m9.figshare.7221605.v2"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.6084/m9.figshare.7221605.v2"}],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Figshare"},"relevantdate":[],"resourcetype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Percutaneous Management of Acquired Right Ventricular Outflow Tract Obstruction due to Giant Coronary Vein Graft Aneurysm"}]} -{"author":[{"affiliation":[],"fullname":"Keller, M.","name":"M.","pid":[],"rank":1,"surname":"Keller"},{"affiliation":[],"fullname":"Cioldi, F.","name":"F.","pid":[],"rank":2,"surname":"Cioldi"},{"affiliation":[],"fullname":"Speich, S.","name":"S.","pid":[],"rank":3,"surname":"Speich"},{"affiliation":[],"fullname":"Meile, R.","name":"R.","pid":[],"rank":4,"surname":"Meile"},{"affiliation":[],"fullname":"Fischer, C.","name":"C.","pid":[],"rank":5,"surname":"Fischer"},{"affiliation":[],"fullname":"Herold-Bonardi, A.","name":"A.","pid":[],"rank":6,"surname":"Herold-Bonardi"},{"affiliation":[],"fullname":"Huber, M.","name":"M.","pid":[],"rank":7,"surname":"Huber"},{"affiliation":[],"fullname":"Vidondo, B.","name":"B.","pid":[],"rank":8,"surname":"Vidondo"},{"affiliation":[],"fullname":"Traub, B.","name":"B.","pid":[],"rank":9,"surname":"Traub"},{"affiliation":[],"fullname":"Abegg, M.","name":"M.","pid":[],"rank":10,"surname":"Abegg"},{"affiliation":[],"fullname":"Brändli, U.-B.","name":"U. -B","pid":[],"rank":11,"surname":"Brändli"},{"affiliation":[],"fullname":"Rösler, E.","name":"E.","pid":[],"rank":12,"surname":"Rösler"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014-01-01"},"dateofcollection":"","dateoftransformation":"","description":[],"embargoenddate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014-11-06"},"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|datacite____::13a6639c122766eb574fb4cbff18c493","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014-01-01"},"distributionlocation":"","hostedby":{"key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.21258/1014641"]}],"language":{"classid":"de-CH","classname":"de-CH","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594988489075,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"mongodb%3A%2F%2Fbeta.services.openaire.eu%3A27017","datestamp":"","harvestDate":"2020-06-30T13:54:53.277Z","identifier":"10.21258/1014641","metadataNamespace":""}},"originalId":["datacite____::13a6639c122766eb574fb4cbff18c493"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.21258/1014641"}],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Birmensdorf, Eidg. Forschungsanstalt WSL"},"relevantdate":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"storagedate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014"},"subject":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Schweizerisches Landesforstinventar - Ergebnistabelle Nr. 144032"}],"version":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"None"}} -{"author":[{"affiliation":[],"fullname":"Unbekannt","name":"","pid":[],"rank":1,"surname":""}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"1933-01-01"},"dateofcollection":"","dateoftransformation":"","description":[],"embargoenddate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"1933-01-01"},"externalReference":[],"extraInfo":[],"format":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"JPEG-Bild"}],"fulltext":[],"geolocation":[],"id":"50|datacite____::b91ffc42a22a193f984fab20ef25e87f","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"1933-01-01"},"distributionlocation":"","hostedby":{"key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"instancetype":{"classid":"0025","classname":"Image","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.3932/ethz-a-000137542"]}],"language":{"classid":"deu/ger","classname":"German","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594989001669,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"mongodb%3A%2F%2Fbeta.services.openaire.eu%3A27017","datestamp":"","harvestDate":"2020-06-30T21:19:46.855Z","identifier":"10.3932/ethz-a-000137542","metadataNamespace":""}},"originalId":["datacite____::b91ffc42a22a193f984fab20ef25e87f"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.3932/ethz-a-000137542"}],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"ETH-Bibliothek Zürich, Bildarchiv"},"relevantdate":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"storagedate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"1933"},"subject":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Napoli, Interno Villa Comunale"}],"version":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"None"}} -{"author":[{"affiliation":[],"fullname":"Occdownload Gbif.Org","name":"","pid":[],"rank":1,"surname":""}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"dateofcollection":"","dateoftransformation":"","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"A dataset containing 1272 species occurrences available in GBIF matching the query: DatasetKey: (Table 2) Stratigraphic ranges of selected Oligocene planktonic foraminifers in ODP Hole 115-706A. The dataset includes 1272 records from 1 constituent datasets: \n 1272 records from (Table 2) Stratigraphic ranges of selected Oligocene planktonic foraminifers in ODP Hole 115-706A. Data from some individual datasets included in this download may be licensed under less restrictive terms."}],"externalReference":[],"extraInfo":[],"format":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Darwin Core Archive"}],"fulltext":[],"geolocation":[],"id":"50|datacite____::b49ea5bf77f4981ac7fdd4f9b37d561b","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"distributionlocation":"","hostedby":{"key":"10|re3data_____::194f60618405f8d2dc58ea68d968a104","value":"Global Biodiversity Information Facility"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"license":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"http://creativecommons.org/licenses/by/4.0/legalcode"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.15468/dl.tygzvu"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastmetadataupdate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-09-15"},"lastupdatetimestamp":1594988854608,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"mongodb%3A%2F%2Fbeta.services.openaire.eu%3A27017","datestamp":"","harvestDate":"2020-06-30T21:08:46.487Z","identifier":"10.15468/dl.tygzvu","metadataNamespace":""}},"originalId":["datacite____::b49ea5bf77f4981ac7fdd4f9b37d561b"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.15468/dl.tygzvu"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"GBIF","classname":"Global Biodiversity Information Facility","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"0006314-160910150852091"}],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"The Global Biodiversity Information Facility"},"relevantdate":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"size":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"59123"},"source":[],"storagedate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016"},"subject":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"GBIF Occurrence Download"}],"version":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"None"}} -{"author":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|opendoar____::2723d092b63885e0d7c260cc007e8b9d","value":"Apollo"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2003-02-01"},"dateofcollection":"2017-02-23T12:37:53.135Z","dateoftransformation":"2020-05-10T08:03:37.351Z","description":[],"externalReference":[],"extraInfo":[],"format":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"6448 bytes"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"6076 bytes"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"chemical/x-cml"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"chemical/x-cml"}],"fulltext":[],"geolocation":[],"id":"50|od_______109::9715bd4f12cff1df00ffaa2febc9f0a2","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|opendoar____::2723d092b63885e0d7c260cc007e8b9d","value":"Apollo"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2003-02-01"},"distributionlocation":"","hostedby":{"key":"10|opendoar____::2723d092b63885e0d7c260cc007e8b9d","value":"Apollo"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://www.dspace.cam.ac.uk/handle/1810/62982"]}],"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594988151792,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"https://www.repository.cam.ac.uk/oai/request","datestamp":"2005-08-23T20:34:18Z","harvestDate":"2017-02-23T12:37:53.135Z","identifier":"oai:www.repository.cam.ac.uk:1810/62982","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"originalId":["od_______109::9715bd4f12cff1df00ffaa2febc9f0a2"],"pid":[],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Unilever Center for Molecular Informatics, Cambridge University"},"relevantdate":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"NSC70052"}]} -{"author":[{"affiliation":[],"fullname":"Schrijer, E.","name":"E.","pid":[],"rank":1,"surname":"Schrijer"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"","dateoftransformation":"2020-06-16T14:47:48.881Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"De weinige aangetroffen sporen in beide proefsleuven van vindplaats 10 zijn alle van vrij recente datum. Afgezien van een vuurstenen schrabber bevatten beide werkputten verder geen archeologische indicatoren van grotere ouderdom. De landschappelijke omstandigheden ter plaatse zullen niet aantrekkelijk zijn geweest voor mensen om zich te vestigen of voor het ontplooien van agrarische activiteiten.\nWel is op de vindplaats een dierbegraving van een klein paard met foetus blootgelegd, alsmede een tweetal mogelijke paalgaten.\n\nOp vindplaats 11 zijn antropogene sporen aangetroffen, bestaande uit een zespalige spieker en een waterkuil die beide dateren uit de Vroege IJzertijd. Deze sporen wijzen erop dat hier de periferie van een erf of nederzetting is aangetroffen. Tijdens de gehele IJzertijd komt bewoning voor in de vorm van zogeheten zwervende erven. Deze erven bestaan uit clusters van een of enkele woonhuizen met bijbehorende stallen, schuren en spiekers. Deze clusters verplaatsen zich door de tijd.\nDe antropogene sporen zijn duidelijk leesbaar, maar een bijbehorende vondstlaag ontbrak. In de kern van de nederzetting zal zich meer archeologisch materiaal bevinden.\nDe op vindplaats 11 aangetroffen spieker en waterkuil blijken op de flank van een oeverwal te liggen. Het bijbehorende woonhuis ligt zeer waarschijnlijk op de kop van die oeverwal.\nBooronderzoek heeft ten westen van put 1 komafzettingen aangetoond. Het is niet voor de hand liggend verder naar het westen antropogene sporen te verwachten. De vindplaats zal voor het grootste deel onder de A2 liggen; het mag worden aangenomen dat de nederzettingsresten bij de aanleg van de huidige A2 zijn vernietigd. Het is mogelijk dat er nog een deel van de periferie bewaard gebleven is aan de oostzijde van de A2."}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::c33033403911286303171564f231c34b","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"distributionlocation":"","hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.17026/dans-xjf-f6sr"]}],"language":{"classid":"dut/nld","classname":"Dutch; Flemish","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594987415271,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2020-06-02T13:42:03Z","harvestDate":"2020-06-16T13:33:24.929Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:33578","metadataNamespace":""}},"originalId":["DansKnawCris::c33033403911286303171564f231c34b"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-xjf-f6sr"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-ngd-kw0"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-xjf-f6sr"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-ngd-kw0"}],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Project Verbreding A2: onderzoek vindplaatsen 10 en 11"}]} \ No newline at end of file +{"author":[{"affiliation":[],"fullname":"Daniels, Matthew J.","name":"Matthew J.","pid":[],"rank":1,"surname":"Daniels"},{"affiliation":[],"fullname":"Newton, James D.","name":"James D.","pid":[],"rank":2,"surname":"Newton"},{"affiliation":[],"fullname":"Kelion, Andrew D.","name":"Andrew D.","pid":[],"rank":3,"surname":"Kelion"},{"affiliation":[],"fullname":"Petrou, Mario","name":"Mario","pid":[],"rank":4,"surname":"Petrou"},{"affiliation":[],"fullname":"Ormerod, Oliver J.","name":"Oliver J.","pid":[],"rank":5,"surname":"Ormerod"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|re3data_____::7980778c78fb4cf0fab13ce2159030dc","value":"figshare"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2018-01-01"},"dateofcollection":"","dateoftransformation":"2020-06-19T22:37:45.636Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Percutaneous Management of Acquired Right Ventricular Outflow Tract Obstruction due to Giant Coronary Vein Graft Aneurysm"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|r37980778c78::cddbf68617b0ae86a263f25b01d1d06d","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|re3data_____::7980778c78fb4cf0fab13ce2159030dc","value":"figshare"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2018-01-01"},"distributionlocation":"","hostedby":{"key":"10|re3data_____::7980778c78fb4cf0fab13ce2159030dc","value":"figshare"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.6084/m9.figshare.7221605.v2","https://figshare.com/articles/Percutaneous_management_of_acquired_right_ventricular_outflow_tract_obstruction_due_to_giant_coronary_vein_graft_aneurysm/7221605"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594988586710,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"https%3A%2F%2Fapi.figshare.com%2Fv2%2Foai","datestamp":"2018-10-17T21:29:18Z","harvestDate":"2020-06-19T13:26:40.229Z","identifier":"oai:figshare.com:article/7221605","metadataNamespace":""}},"originalId":["r37980778c78::cddbf68617b0ae86a263f25b01d1d06d"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.6084/m9.figshare.7221605.v2"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.6084/m9.figshare.7221605.v2"}],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Figshare"},"relevantdate":[],"resourcetype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Percutaneous Management of Acquired Right Ventricular Outflow Tract Obstruction due to Giant Coronary Vein Graft Aneurysm"}]} +{"author":[{"affiliation":[],"fullname":"Keller, M.","name":"M.","pid":[],"rank":1,"surname":"Keller"},{"affiliation":[],"fullname":"Cioldi, F.","name":"F.","pid":[],"rank":2,"surname":"Cioldi"},{"affiliation":[],"fullname":"Speich, S.","name":"S.","pid":[],"rank":3,"surname":"Speich"},{"affiliation":[],"fullname":"Meile, R.","name":"R.","pid":[],"rank":4,"surname":"Meile"},{"affiliation":[],"fullname":"Fischer, C.","name":"C.","pid":[],"rank":5,"surname":"Fischer"},{"affiliation":[],"fullname":"Herold-Bonardi, A.","name":"A.","pid":[],"rank":6,"surname":"Herold-Bonardi"},{"affiliation":[],"fullname":"Huber, M.","name":"M.","pid":[],"rank":7,"surname":"Huber"},{"affiliation":[],"fullname":"Vidondo, B.","name":"B.","pid":[],"rank":8,"surname":"Vidondo"},{"affiliation":[],"fullname":"Traub, B.","name":"B.","pid":[],"rank":9,"surname":"Traub"},{"affiliation":[],"fullname":"Abegg, M.","name":"M.","pid":[],"rank":10,"surname":"Abegg"},{"affiliation":[],"fullname":"Brändli, U.-B.","name":"U. -B","pid":[],"rank":11,"surname":"Brändli"},{"affiliation":[],"fullname":"Rösler, E.","name":"E.","pid":[],"rank":12,"surname":"Rösler"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014-01-01"},"dateofcollection":"","dateoftransformation":"","description":[],"embargoenddate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014-11-06"},"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|datacite____::13a6639c122766eb574fb4cbff18c493","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014-01-01"},"distributionlocation":"","hostedby":{"key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.21258/1014641"]}],"language":{"classid":"de-CH","classname":"de-CH","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594988489075,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"mongodb%3A%2F%2Fbeta.services.openaire.eu%3A27017","datestamp":"","harvestDate":"2020-06-30T13:54:53.277Z","identifier":"10.21258/1014641","metadataNamespace":""}},"originalId":["datacite____::13a6639c122766eb574fb4cbff18c493"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.21258/1014641"}],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Birmensdorf, Eidg. Forschungsanstalt WSL"},"relevantdate":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"storagedate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014"},"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Schweizerisches Landesforstinventar - Ergebnistabelle Nr. 144032"}],"version":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"None"}} +{"author":[{"affiliation":[],"fullname":"Unbekannt","name":"","pid":[],"rank":1,"surname":""}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"1933-01-01"},"dateofcollection":"","dateoftransformation":"","description":[],"embargoenddate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"1933-01-01"},"externalReference":[],"extraInfo":[],"format":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"JPEG-Bild"}],"fulltext":[],"geolocation":[],"id":"50|datacite____::b91ffc42a22a193f984fab20ef25e87f","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"1933-01-01"},"distributionlocation":"","hostedby":{"key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"instancetype":{"classid":"0025","classname":"Image","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.3932/ethz-a-000137542"]}],"language":{"classid":"deu/ger","classname":"German","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594989001669,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"mongodb%3A%2F%2Fbeta.services.openaire.eu%3A27017","datestamp":"","harvestDate":"2020-06-30T21:19:46.855Z","identifier":"10.3932/ethz-a-000137542","metadataNamespace":""}},"originalId":["datacite____::b91ffc42a22a193f984fab20ef25e87f"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.3932/ethz-a-000137542"}],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"ETH-Bibliothek Zürich, Bildarchiv"},"relevantdate":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"storagedate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"1933"},"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Napoli, Interno Villa Comunale"}],"version":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"None"}} +{"author":[{"affiliation":[],"fullname":"Occdownload Gbif.Org","name":"","pid":[],"rank":1,"surname":""}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"dateofcollection":"","dateoftransformation":"","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"A dataset containing 1272 species occurrences available in GBIF matching the query: DatasetKey: (Table 2) Stratigraphic ranges of selected Oligocene planktonic foraminifers in ODP Hole 115-706A. The dataset includes 1272 records from 1 constituent datasets: \n 1272 records from (Table 2) Stratigraphic ranges of selected Oligocene planktonic foraminifers in ODP Hole 115-706A. Data from some individual datasets included in this download may be licensed under less restrictive terms."}],"externalReference":[],"extraInfo":[],"format":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Darwin Core Archive"}],"fulltext":[],"geolocation":[],"id":"50|datacite____::b49ea5bf77f4981ac7fdd4f9b37d561b","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"distributionlocation":"","hostedby":{"key":"10|re3data_____::194f60618405f8d2dc58ea68d968a104","value":"Global Biodiversity Information Facility"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"license":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"http://creativecommons.org/licenses/by/4.0/legalcode"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.15468/dl.tygzvu"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastmetadataupdate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-09-15"},"lastupdatetimestamp":1594988854608,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"mongodb%3A%2F%2Fbeta.services.openaire.eu%3A27017","datestamp":"","harvestDate":"2020-06-30T21:08:46.487Z","identifier":"10.15468/dl.tygzvu","metadataNamespace":""}},"originalId":["datacite____::b49ea5bf77f4981ac7fdd4f9b37d561b"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.15468/dl.tygzvu"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"GBIF","classname":"Global Biodiversity Information Facility","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"0006314-160910150852091"}],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"The Global Biodiversity Information Facility"},"relevantdate":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"size":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"59123"},"source":[],"storagedate":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016"},"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"GBIF Occurrence Download"}],"version":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"None"}} +{"author":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|opendoar____::2723d092b63885e0d7c260cc007e8b9d","value":"Apollo"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2003-02-01"},"dateofcollection":"2017-02-23T12:37:53.135Z","dateoftransformation":"2020-05-10T08:03:37.351Z","description":[],"externalReference":[],"extraInfo":[],"format":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"6448 bytes"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"6076 bytes"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"chemical/x-cml"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"chemical/x-cml"}],"fulltext":[],"geolocation":[],"id":"50|od_______109::9715bd4f12cff1df00ffaa2febc9f0a2","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|opendoar____::2723d092b63885e0d7c260cc007e8b9d","value":"Apollo"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2003-02-01"},"distributionlocation":"","hostedby":{"key":"10|opendoar____::2723d092b63885e0d7c260cc007e8b9d","value":"Apollo"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://www.dspace.cam.ac.uk/handle/1810/62982"]}],"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594988151792,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"https://www.repository.cam.ac.uk/oai/request","datestamp":"2005-08-23T20:34:18Z","harvestDate":"2017-02-23T12:37:53.135Z","identifier":"oai:www.repository.cam.ac.uk:1810/62982","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"originalId":["od_______109::9715bd4f12cff1df00ffaa2febc9f0a2"],"pid":[],"publisher":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Unilever Center for Molecular Informatics, Cambridge University"},"relevantdate":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"NSC70052"}]} +{"author":[{"affiliation":[],"fullname":"Schrijer, E.","name":"E.","pid":[],"rank":1,"surname":"Schrijer"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"","dateoftransformation":"2020-06-16T14:47:48.881Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"De weinige aangetroffen sporen in beide proefsleuven van vindplaats 10 zijn alle van vrij recente datum. Afgezien van een vuurstenen schrabber bevatten beide werkputten verder geen archeologische indicatoren van grotere ouderdom. De landschappelijke omstandigheden ter plaatse zullen niet aantrekkelijk zijn geweest voor mensen om zich te vestigen of voor het ontplooien van agrarische activiteiten.\nWel is op de vindplaats een dierbegraving van een klein paard met foetus blootgelegd, alsmede een tweetal mogelijke paalgaten.\n\nOp vindplaats 11 zijn antropogene sporen aangetroffen, bestaande uit een zespalige spieker en een waterkuil die beide dateren uit de Vroege IJzertijd. Deze sporen wijzen erop dat hier de periferie van een erf of nederzetting is aangetroffen. Tijdens de gehele IJzertijd komt bewoning voor in de vorm van zogeheten zwervende erven. Deze erven bestaan uit clusters van een of enkele woonhuizen met bijbehorende stallen, schuren en spiekers. Deze clusters verplaatsen zich door de tijd.\nDe antropogene sporen zijn duidelijk leesbaar, maar een bijbehorende vondstlaag ontbrak. In de kern van de nederzetting zal zich meer archeologisch materiaal bevinden.\nDe op vindplaats 11 aangetroffen spieker en waterkuil blijken op de flank van een oeverwal te liggen. Het bijbehorende woonhuis ligt zeer waarschijnlijk op de kop van die oeverwal.\nBooronderzoek heeft ten westen van put 1 komafzettingen aangetoond. Het is niet voor de hand liggend verder naar het westen antropogene sporen te verwachten. De vindplaats zal voor het grootste deel onder de A2 liggen; het mag worden aangenomen dat de nederzettingsresten bij de aanleg van de huidige A2 zijn vernietigd. Het is mogelijk dat er nog een deel van de periferie bewaard gebleven is aan de oostzijde van de A2."}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::c33033403911286303171564f231c34b","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"distributionlocation":"","hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["http://dx.doi.org/10.17026/dans-xjf-f6sr"]}],"language":{"classid":"dut/nld","classname":"Dutch; Flemish","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1594987415271,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2020-06-02T13:42:03Z","harvestDate":"2020-06-16T13:33:24.929Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:33578","metadataNamespace":""}},"originalId":["DansKnawCris::c33033403911286303171564f231c34b"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-xjf-f6sr"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-ngd-kw0"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-xjf-f6sr"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-ngd-kw0"}],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Project Verbreding A2: onderzoek vindplaatsen 10 en 11"}]} \ No newline at end of file From 5fb58362c58a5d77723f283142a90da924ac5a45 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 4 Aug 2023 17:18:15 +0200 Subject: [PATCH 09/12] moved parameter file. Added 40| as prefix on projects for computing the delta --- .../dhp/oa/graph/dump/complete/QueryInformationSystem.java | 5 ++++- .../graph/dump/projectssubset/ProjectsSubsetSparkJob.java | 6 +++++- .../{common => oa/graph/dump}/input_maketar_parameters.json | 0 3 files changed, 9 insertions(+), 2 deletions(-) rename dump/src/main/resources/eu/dnetlib/dhp/{common => oa/graph/dump}/input_maketar_parameters.json (100%) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java index d78370a..262fe1d 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java @@ -176,7 +176,10 @@ public class QueryInformationSystem { for (Object node : el.selectNodes(".//param")) { Node n = (Node) node; if (n.valueOf("./@name").equals("openaireId")) { - return prefix + "|" + n.getText(); + String id = n.getText(); + if (id.startsWith(prefix + "|")) + return id; + return prefix + "|" + id; } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/projectssubset/ProjectsSubsetSparkJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/projectssubset/ProjectsSubsetSparkJob.java index fb94eba..7f83d3c 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/projectssubset/ProjectsSubsetSparkJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/projectssubset/ProjectsSubsetSparkJob.java @@ -58,7 +58,11 @@ public class ProjectsSubsetSparkJob implements Serializable { String projectListPath) { Dataset projectList = spark.read().textFile(projectListPath); Dataset projects; - projects = Utils.readPath(spark, inputPath, Project.class); + projects = Utils.readPath(spark, inputPath, Project.class) + .map((MapFunction) p -> { + p.setId("40|" + p.getId()); + return p; + }, Encoders.bean(Project.class)); projects .joinWith(projectList, projects.col("id").equalTo(projectList.col("value")), "left") .map((MapFunction, Project>) t2 -> { diff --git a/dump/src/main/resources/eu/dnetlib/dhp/common/input_maketar_parameters.json b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_maketar_parameters.json similarity index 100% rename from dump/src/main/resources/eu/dnetlib/dhp/common/input_maketar_parameters.json rename to dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_maketar_parameters.json From e9aca6b702006a3527fc053ea57d9cc9e4484df8 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 4 Aug 2023 19:32:16 +0200 Subject: [PATCH 10/12] refactoring --- .../SparkSelectValidRelationsJob.java | 115 ++++-------------- 1 file changed, 24 insertions(+), 91 deletions(-) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java index 73ebc21..d188685 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java @@ -4,15 +4,18 @@ package eu.dnetlib.dhp.oa.graph.dump.complete; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Optional; + import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; -import org.apache.spark.api.java.function.FilterFunction; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; + +import org.apache.spark.sql.*; + +import org.apache.spark.sql.types.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,99 +63,29 @@ public class SparkSelectValidRelationsJob implements Serializable { isSparkSessionManaged, spark -> { Utils.removeOutputDir(spark, outputPath); - selectValidRelation(spark, inputPath, outputPath); + selectValidRelation2(spark, inputPath, outputPath); }); } - private static void selectValidRelation(SparkSession spark, String inputPath, String outputPath) { - Dataset relation = Utils - .readPath(spark, inputPath + "/relation", Relation.class) - .filter((FilterFunction) r -> !r.getDataInfo().getDeletedbyinference()); - Dataset publication = Utils - .readPath(spark, inputPath + "/publication", Publication.class) - .filter( - (FilterFunction) p -> !p.getDataInfo().getDeletedbyinference() - && !p.getDataInfo().getInvisible()); - Dataset dataset = Utils - .readPath(spark, inputPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class) - .filter( - (FilterFunction) d -> !d.getDataInfo().getDeletedbyinference() - && !d.getDataInfo().getInvisible()); - ; - Dataset software = Utils - .readPath(spark, inputPath + "/software", Software.class) - .filter( - (FilterFunction) s -> !s.getDataInfo().getDeletedbyinference() - && !s.getDataInfo().getInvisible()); - Dataset other = Utils - .readPath(spark, inputPath + "/otherresearchproduct", OtherResearchProduct.class) - .filter( - (FilterFunction) o -> !o.getDataInfo().getDeletedbyinference() - && !o.getDataInfo().getInvisible()); - Dataset organization = Utils - .readPath(spark, inputPath + "/organization", Organization.class) - .filter( - (FilterFunction) o -> !o.getDataInfo().getDeletedbyinference() - && !o.getDataInfo().getInvisible()); - Dataset project = Utils - .readPath(spark, inputPath + "/project", Project.class) - .filter( - (FilterFunction) p -> !p.getDataInfo().getDeletedbyinference() - && !p.getDataInfo().getInvisible()); - Dataset datasource = Utils - .readPath(spark, inputPath + "/datasource", Datasource.class) - .filter( - (FilterFunction) d -> !d.getDataInfo().getDeletedbyinference() - && !d.getDataInfo().getInvisible()); + private static void selectValidRelation2(SparkSession spark, String inputPath, String outputPath){ + final StructType structureSchema = new StructType().fromDDL("`id` STRING, `dataInfo` STRUCT<`deletedbyinference`:BOOLEAN,`invisible`:BOOLEAN>"); - relation.createOrReplaceTempView("relation"); - publication.createOrReplaceTempView("publication"); - dataset.createOrReplaceTempView("dataset"); - other.createOrReplaceTempView("other"); - software.createOrReplaceTempView("software"); - organization.createOrReplaceTempView("organization"); - project.createOrReplaceTempView("project"); - datasource.createOrReplaceTempView("datasource"); + org.apache.spark.sql.Dataset df =spark.createDataFrame(new ArrayList(), structureSchema); + List entities = Arrays.asList("publication", "dataset","otherresearchproduct","software","organization","project","datasource"); + for(String e : entities) + df = df.union(spark.read().schema(structureSchema).json(inputPath + "/" + e).filter("dataInfo.deletedbyinference != true")); - spark - .sql( - "SELECT id " + - "FROM publication " + - "UNION ALL " + - "SELECT id " + - "FROM dataset " + - "UNION ALL " + - "SELECT id " + - "FROM other " + - "UNION ALL " + - "SELECT id " + - "FROM software " + - "UNION ALL " + - "SELECT id " + - "FROM organization " + - "UNION ALL " + - "SELECT id " + - "FROM project " + - "UNION ALL " + - "SELECT id " + - "FROM datasource ") - .createOrReplaceTempView("identifiers"); - - spark - .sql( - "SELECT relation.* " + - "FROM relation " + - "JOIN identifiers i1 " + - "ON source = i1.id " + - "JOIN identifiers i2 " + - "ON target = i2.id ") - .as(Encoders.bean(Relation.class)) - .write() - .option("compression", "gzip") - .mode(SaveMode.Overwrite) - .json(outputPath); + org.apache.spark.sql.Dataset relations = spark.read().schema(Encoders.bean(Relation.class).schema()).json(inputPath + "/relation") + .filter("dataInfo.deletedbyinference == false"); + relations.join(df, relations.col("source").equalTo(df.col("id")), "leftsemi") + .join(df, relations.col("target").equalTo(df.col("id")), "leftsemi") + .write() + .mode(SaveMode.Overwrite) + .option("compression","gzip") + .json(outputPath); } + } From 24be522e7c8be704f015132e2ba519536a459a76 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 7 Aug 2023 13:56:58 +0200 Subject: [PATCH 11/12] fixed NPE, moved class to generate tar --- .../dhp/{ => oa}/common/MakeTarArchive.java | 2 +- .../eu/dnetlib/dhp/oa/graph/dump/MakeTar.java | 2 +- .../dhp/oa/graph/dump/ResultMapper.java | 20 ++++++--- .../dump/complete/QueryInformationSystem.java | 4 +- .../SparkSelectValidRelationsJob.java | 44 ++++++++++++------- .../ProjectsSubsetSparkJob.java | 11 ++--- 6 files changed, 50 insertions(+), 33 deletions(-) rename dump/src/main/java/eu/dnetlib/dhp/{ => oa}/common/MakeTarArchive.java (99%) diff --git a/dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java b/dump/src/main/java/eu/dnetlib/dhp/oa/common/MakeTarArchive.java similarity index 99% rename from dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java rename to dump/src/main/java/eu/dnetlib/dhp/oa/common/MakeTarArchive.java index 43e61e5..20058bc 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/common/MakeTarArchive.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.common; +package eu.dnetlib.dhp.oa.common; import java.io.BufferedInputStream; import java.io.IOException; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java index cb2e29b..44f4ef6 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java @@ -15,7 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.MakeTarArchive; +import eu.dnetlib.dhp.oa.common.MakeTarArchive; public class MakeTar implements Serializable { diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java index 52584f9..75f01a6 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java @@ -515,14 +515,20 @@ public class ResultMapper implements Serializable { setCommonValue(i, instance); - instance - .setCollectedfrom( - CfHbKeyValue - .newInstance(i.getCollectedfrom().getKey().substring(3), i.getCollectedfrom().getValue())); + if (Optional.ofNullable(i.getCollectedfrom()).isPresent() && + Optional.ofNullable(i.getCollectedfrom().getKey()).isPresent() && + StringUtils.isNotBlank(i.getCollectedfrom().getKey())) + instance + .setCollectedfrom( + CfHbKeyValue + .newInstance(i.getCollectedfrom().getKey().substring(3), i.getCollectedfrom().getValue())); - instance - .setHostedby( - CfHbKeyValue.newInstance(i.getHostedby().getKey().substring(3), i.getHostedby().getValue())); + if (Optional.ofNullable(i.getHostedby()).isPresent() && + Optional.ofNullable(i.getHostedby().getKey()).isPresent() && + StringUtils.isNotBlank(i.getHostedby().getKey())) + instance + .setHostedby( + CfHbKeyValue.newInstance(i.getHostedby().getKey().substring(3), i.getHostedby().getValue())); return instance; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java index 262fe1d..b982b26 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java @@ -219,9 +219,9 @@ public class QueryInformationSystem { } if (funding.toLowerCase().contains("h2020")) { nsp = "corda__h2020::"; - } else if (funding.toLowerCase().contains("he")){ + } else if (funding.toLowerCase().contains("he")) { nsp = "corda_____he::"; - }else{ + } else { nsp = "corda_______::"; } break; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java index d188685..12c2c00 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java @@ -9,12 +9,9 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; - import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; - import org.apache.spark.sql.*; - import org.apache.spark.sql.types.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,23 +66,36 @@ public class SparkSelectValidRelationsJob implements Serializable { } - private static void selectValidRelation2(SparkSession spark, String inputPath, String outputPath){ - final StructType structureSchema = new StructType().fromDDL("`id` STRING, `dataInfo` STRUCT<`deletedbyinference`:BOOLEAN,`invisible`:BOOLEAN>"); + private static void selectValidRelation2(SparkSession spark, String inputPath, String outputPath) { + final StructType structureSchema = new StructType() + .fromDDL("`id` STRING, `dataInfo` STRUCT<`deletedbyinference`:BOOLEAN,`invisible`:BOOLEAN>"); - org.apache.spark.sql.Dataset df =spark.createDataFrame(new ArrayList(), structureSchema); - List entities = Arrays.asList("publication", "dataset","otherresearchproduct","software","organization","project","datasource"); - for(String e : entities) - df = df.union(spark.read().schema(structureSchema).json(inputPath + "/" + e).filter("dataInfo.deletedbyinference != true")); + org.apache.spark.sql.Dataset df = spark.createDataFrame(new ArrayList(), structureSchema); + List entities = Arrays + .asList( + "publication", "dataset", "otherresearchproduct", "software", "organization", "project", "datasource"); + for (String e : entities) + df = df + .union( + spark + .read() + .schema(structureSchema) + .json(inputPath + "/" + e) + .filter("dataInfo.deletedbyinference != true and dataInfo.invisible != true")); - org.apache.spark.sql.Dataset relations = spark.read().schema(Encoders.bean(Relation.class).schema()).json(inputPath + "/relation") - .filter("dataInfo.deletedbyinference == false"); + org.apache.spark.sql.Dataset relations = spark + .read() + .schema(Encoders.bean(Relation.class).schema()) + .json(inputPath + "/relation") + .filter("dataInfo.deletedbyinference == false"); - relations.join(df, relations.col("source").equalTo(df.col("id")), "leftsemi") - .join(df, relations.col("target").equalTo(df.col("id")), "leftsemi") - .write() - .mode(SaveMode.Overwrite) - .option("compression","gzip") - .json(outputPath); + relations + .join(df, relations.col("source").equalTo(df.col("id")), "leftsemi") + .join(df, relations.col("target").equalTo(df.col("id")), "leftsemi") + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/projectssubset/ProjectsSubsetSparkJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/projectssubset/ProjectsSubsetSparkJob.java index 7f83d3c..b95312f 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/projectssubset/ProjectsSubsetSparkJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/projectssubset/ProjectsSubsetSparkJob.java @@ -58,11 +58,12 @@ public class ProjectsSubsetSparkJob implements Serializable { String projectListPath) { Dataset projectList = spark.read().textFile(projectListPath); Dataset projects; - projects = Utils.readPath(spark, inputPath, Project.class) - .map((MapFunction) p -> { - p.setId("40|" + p.getId()); - return p; - }, Encoders.bean(Project.class)); + projects = Utils + .readPath(spark, inputPath, Project.class) + .map((MapFunction) p -> { + p.setId("40|" + p.getId()); + return p; + }, Encoders.bean(Project.class)); projects .joinWith(projectList, projects.col("id").equalTo(projectList.col("value")), "left") .map((MapFunction, Project>) t2 -> { From d1f41b8e284c03c4617dda9c7d6f740a4cde789c Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 17 Aug 2023 10:14:20 +0200 Subject: [PATCH 12/12] removed organization without legalname and legalshortname from the dump --- .../eu/dnetlib/dhp/ExecCreateSchemas.java | 1 + .../community_infrastructure_schema.json | 69 +- .../jsonschemas/community_result_schema.json | 621 ++++++++++++ .../jsonschemas/datasource_schema.json | 286 +++--- .../openaire-community-dump-schema.json | 938 ++++++++++-------- .../jsonschemas/organization_schema.json | 84 +- .../resources/jsonschemas/project_schema.json | 172 ++-- .../jsonschemas/relation_schema.json | 102 +- .../resources/jsonschemas/result_schema.json | 732 +++++++------- .../graph/dump/community/CommunitySplit.java | 41 +- .../dump/complete/SparkDumpEntitiesJob.java | 4 + .../SparkFindResultsRelatedToCountry.java | 4 +- .../funderresults/SparkDumpFunderResults.java | 39 +- .../countryresults/oozie_app/workflow.xml | 32 +- .../funder/oozie_app/workflow.xml | 3 +- .../subset/oozie_app/workflow.xml | 1 + .../oa/graph/dump/SplitForCommunityTest.java | 1 + .../oa/graph/dump/communityResult/publication | 2 +- 18 files changed, 1916 insertions(+), 1216 deletions(-) create mode 100644 dump-schema/src/main/resources/jsonschemas/community_result_schema.json diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/ExecCreateSchemas.java b/dump-schema/src/main/java/eu/dnetlib/dhp/ExecCreateSchemas.java index a9a0c49..b669da2 100644 --- a/dump-schema/src/main/java/eu/dnetlib/dhp/ExecCreateSchemas.java +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/ExecCreateSchemas.java @@ -40,6 +40,7 @@ public class ExecCreateSchemas { .get(Paths.get(getClass().getResource("/").getPath()).toAbsolutePath() + directory) .toString(); + System.out.println(dir); if (!Files.exists(Paths.get(dir))) { Files.createDirectories(Paths.get(dir)); } diff --git a/dump-schema/src/main/resources/jsonschemas/community_infrastructure_schema.json b/dump-schema/src/main/resources/jsonschemas/community_infrastructure_schema.json index 727432a..35c69cf 100644 --- a/dump-schema/src/main/resources/jsonschemas/community_infrastructure_schema.json +++ b/dump-schema/src/main/resources/jsonschemas/community_infrastructure_schema.json @@ -1,35 +1,38 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "acronym": { - "type": "string", - "description": "The acronym of the community" - }, - "description": { - "type": "string", - "description": "Description of the research community/research infrastructure" - }, - "id": { - "type": "string", - "description": "OpenAIRE id of the research community/research infrastructure" - }, - "name": { - "type": "string", - "description": "The long name of the community" - }, - "subject": { - "description": "Only for research communities: the list of the subjects associated to the research community", - "type": "array", - "items": {"type": "string"} - }, - "type": { - "type": "string", - "description": "One of {Research Community, Research infrastructure}" - }, - "zenodo_community": { - "type": "string", - "description": "The URL of the Zenodo community associated to the Research community/Research infrastructure" - } + "$schema" : "http://json-schema.org/draft-07/schema#", + "type" : "object", + "properties" : { + "acronym" : { + "type" : "string", + "description" : "The acronym of the community" + }, + "description" : { + "type" : "string", + "description" : "Description of the research community/research infrastructure" + }, + "id" : { + "type" : "string", + "description" : "The OpenAIRE id for the community/research infrastructure" + }, + "name" : { + "type" : "string", + "description" : "The long name of the community" + }, + "subject" : { + "description" : "Only for research communities: the list of the subjects associated to the research community", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for research communities: the list of the subjects associated to the research community" + } + }, + "type" : { + "type" : "string", + "description" : "One of {Research Community, Research infrastructure}" + }, + "zenodo_community" : { + "type" : "string", + "description" : "The URL of the Zenodo community associated to the Research community/Research infrastructure" } -} \ No newline at end of file + } +} diff --git a/dump-schema/src/main/resources/jsonschemas/community_result_schema.json b/dump-schema/src/main/resources/jsonschemas/community_result_schema.json new file mode 100644 index 0000000..a30fe6a --- /dev/null +++ b/dump-schema/src/main/resources/jsonschemas/community_result_schema.json @@ -0,0 +1,621 @@ +{ + "$schema" : "http://json-schema.org/draft-07/schema#", + "definitions" : { + "CfHbKeyValue" : { + "type" : "object", + "properties" : { + "key" : { + "type" : "string", + "description" : "the OpenAIRE identifier of the data source" + }, + "value" : { + "type" : "string", + "description" : "the name of the data source" + } + } + }, + "Provenance" : { + "type" : "object", + "properties" : { + "provenance" : { + "type" : "string" + }, + "trust" : { + "type" : "string" + } + } + }, + "ResultPid" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme of the persistent identifier for the result (i.e. doi). If the pid is here it means the information for the pid has been collected from an authority for that pid type (i.e. Crossref/Datacite for doi). The set of authoritative pid is: doi when collected from Crossref or Datacite pmid when collected from EuroPubmed, arxiv when collected from arXiv, handle from the repositories" + }, + "value" : { + "type" : "string", + "description" : "The value expressed in the scheme (i.e. 10.1000/182)" + } + } + } + }, + "type" : "object", + "properties" : { + "author" : { + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "fullname" : { + "type" : "string" + }, + "name" : { + "type" : "string" + }, + "pid" : { + "type" : "object", + "properties" : { + "id" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The author's pid scheme. OpenAIRE currently supports 'ORCID'" + }, + "value" : { + "type" : "string", + "description" : "The author's pid value in that scheme (i.e. 0000-1111-2222-3333)" + } + } + }, + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "The reason why the pid was associated to the author" + } ] + } + }, + "description" : "The author's persistent identifiers" + }, + "rank" : { + "type" : "integer" + }, + "surname" : { + "type" : "string" + } + } + } + }, + "bestaccessright" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" + }, + "label" : { + "type" : "string", + "description" : "Label for the access mode" + }, + "scheme" : { + "type" : "string", + "description" : "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" + } + }, + "description" : "The openest of the access rights of this result." + }, + "codeRepositoryUrl" : { + "type" : "string", + "description" : "Only for results with type 'software': the URL to the repository with the source code" + }, + "collectedfrom" : { + "description" : "Information about the sources from which the record has been collected", + "type" : "array", + "items" : { + "allOf" : [ { + "$ref" : "#/definitions/CfHbKeyValue" + }, { + "description" : "Information about the sources from which the record has been collected" + } ] + } + }, + "contactgroup" : { + "description" : "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource" + } + }, + "contactperson" : { + "description" : "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource" + } + }, + "container" : { + "type" : "object", + "properties" : { + "conferencedate" : { + "type" : "string" + }, + "conferenceplace" : { + "type" : "string" + }, + "edition" : { + "type" : "string", + "description" : "Edition of the journal or conference proceeding" + }, + "ep" : { + "type" : "string", + "description" : "End page" + }, + "iss" : { + "type" : "string", + "description" : "Journal issue number" + }, + "issnLinking" : { + "type" : "string" + }, + "issnOnline" : { + "type" : "string" + }, + "issnPrinted" : { + "type" : "string" + }, + "name" : { + "type" : "string", + "description" : "Name of the journal or conference" + }, + "sp" : { + "type" : "string", + "description" : "Start page" + }, + "vol" : { + "type" : "string", + "description" : "Volume" + } + }, + "description" : "Container has information about the conference or journal where the result has been presented or published" + }, + "context" : { + "description" : "Reference to a relevant research infrastructure, initiative or community (RI/RC) among those collaborating with OpenAIRE. Please see https://connect.openaire.eu", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "Code identifying the RI/RC" + }, + "label" : { + "type" : "string", + "description" : "Label of the RI/RC" + }, + "provenance" : { + "description" : "Why this result is associated to the RI/RC.", + "type" : "array", + "items" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "Why this result is associated to the RI/RC." + } ] + } + } + }, + "description" : "Reference to a relevant research infrastructure, initiative or community (RI/RC) among those collaborating with OpenAIRE. Please see https://connect.openaire.eu" + } + }, + "contributor" : { + "description" : "Contributors for the result", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Contributors for the result" + } + }, + "country" : { + "description" : "The list of countries associated to this result", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "ISO 3166-1 alpha-2 country code (i.e. IT)" + }, + "label" : { + "type" : "string", + "description" : "The label for that code (i.e. Italy)" + }, + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "Why this result is associated to the country." + } ] + } + }, + "description" : "The list of countries associated to this result" + } + }, + "coverage" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "dateofcollection" : { + "type" : "string", + "description" : "When OpenAIRE collected the record the last time" + }, + "description" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "documentationUrl" : { + "description" : "Only for results with type 'software': URL to the software documentation", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': URL to the software documentation" + } + }, + "embargoenddate" : { + "type" : "string", + "description" : "Date when the embargo ends and this result turns Open Access" + }, + "format" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "geolocation" : { + "description" : "Geolocation information", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "box" : { + "type" : "string" + }, + "place" : { + "type" : "string" + }, + "point" : { + "type" : "string" + } + }, + "description" : "Geolocation information" + } + }, + "id" : { + "type" : "string", + "description" : "The OpenAIRE identifiers for this result" + }, + "indicators" : { + "type" : "object", + "properties" : { + "bipIndicators" : { + "description" : "The impact measures (i.e. popularity)", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "clazz" : { + "type" : "string" + }, + "indicator" : { + "type" : "string" + }, + "score" : { + "type" : "string" + } + }, + "description" : "The impact measures (i.e. popularity)" + } + }, + "usageCounts" : { + "type" : "object", + "properties" : { + "downloads" : { + "type" : "string" + }, + "views" : { + "type" : "string" + } + }, + "description" : "The usage counts (i.e. downloads)" + } + }, + "description" : "Indicators computed for this result, for example UsageCount ones" + }, + "instance" : { + "description" : "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "accessright" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" + }, + "label" : { + "type" : "string", + "description" : "Label for the access mode" + }, + "openAccessRoute" : { + "type" : "string", + "enum" : [ "gold", "green", "hybrid", "bronze" ] + }, + "scheme" : { + "type" : "string", + "description" : "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" + } + }, + "description" : "The accessRights for this materialization of the result" + }, + "alternateIdentifier" : { + "description" : "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme of the identifier. It can be a persistent identifier (i.e. doi). If it is present in the alternate identifiers it means it has not been forged by an authority for that pid. For example we collect metadata from an institutional repository that provides as identifier for the result also the doi" + }, + "value" : { + "type" : "string", + "description" : "The value expressed in the scheme" + } + }, + "description" : "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs" + } + }, + "articleprocessingcharge" : { + "type" : "object", + "properties" : { + "amount" : { + "type" : "string" + }, + "currency" : { + "type" : "string" + } + }, + "description" : "The money spent to make this book or article available in Open Access. Source for this information is the OpenAPC initiative." + }, + "collectedfrom" : { + "allOf" : [ { + "$ref" : "#/definitions/CfHbKeyValue" + }, { + "description" : "Information about the source from which the record has been collected" + } ] + }, + "hostedby" : { + "allOf" : [ { + "$ref" : "#/definitions/CfHbKeyValue" + }, { + "description" : "Information about the source from which the instance can be viewed or downloaded." + } ] + }, + "license" : { + "type" : "string" + }, + "pid" : { + "type" : "array", + "items" : { + "$ref" : "#/definitions/ResultPid" + } + }, + "publicationdate" : { + "type" : "string", + "description" : "Date of the research product" + }, + "refereed" : { + "type" : "string", + "description" : "If this instance has been peer-reviewed or not. Allowed values are peerReviewed, nonPeerReviewed, UNKNOWN (as defined in https://api.openaire.eu/vocabularies/dnet:review_levels)" + }, + "type" : { + "type" : "string", + "description" : "The specific sub-type of this instance (see https://api.openaire.eu/vocabularies/dnet:result_typologies following the links)" + }, + "url" : { + "description" : "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. ", + "type" : "array", + "items" : { + "type" : "string", + "description" : "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. " + } + } + }, + "description" : "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version" + } + }, + "language" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "alpha-3/ISO 639-2 code of the language" + }, + "label" : { + "type" : "string", + "description" : "Language label in English" + } + } + }, + "lastupdatetimestamp" : { + "type" : "integer", + "description" : "Timestamp of last update of the record in OpenAIRE" + }, + "maintitle" : { + "type" : "string", + "description" : "A name or title by which a scientific result is known. May be the title of a publication, of a dataset or the name of a piece of software." + }, + "originalId" : { + "description" : "Identifiers of the record at the original sources", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Identifiers of the record at the original sources" + } + }, + "pid" : { + "description" : "Persistent identifiers of the result", + "type" : "array", + "items" : { + "allOf" : [ { + "$ref" : "#/definitions/ResultPid" + }, { + "description" : "Persistent identifiers of the result" + } ] + } + }, + "programmingLanguage" : { + "type" : "string", + "description" : "Only for results with type 'software': the programming language" + }, + "projects" : { + "description" : "List of projects (i.e. grants) that (co-)funded the production ofn the research results", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "acronym" : { + "type" : "string", + "description" : "The acronym of the project" + }, + "code" : { + "type" : "string", + "description" : "The grant agreement number" + }, + "funder" : { + "type" : "object", + "properties" : { + "fundingStream" : { + "type" : "string", + "description" : "Stream of funding (e.g. for European Commission can be H2020 or FP7)" + }, + "jurisdiction" : { + "type" : "string", + "description" : "Geographical jurisdiction (e.g. for European Commission is EU, for Croatian Science Foundation is HR)" + }, + "name" : { + "type" : "string", + "description" : "The name of the funder (European Commission)" + }, + "shortName" : { + "type" : "string", + "description" : "The short name of the funder (EC)" + } + }, + "description" : "Information about the funder funding the project" + }, + "id" : { + "type" : "string", + "description" : "The OpenAIRE id for the project" + }, + "provenance" : { + "$ref" : "#/definitions/Provenance" + }, + "title" : { + "type" : "string" + }, + "validated" : { + "type" : "object", + "properties" : { + "validatedByFunder" : { + "type" : "boolean" + }, + "validationDate" : { + "type" : "string" + } + } + } + }, + "description" : "List of projects (i.e. grants) that (co-)funded the production ofn the research results" + } + }, + "publicationdate" : { + "type" : "string", + "description" : "Main date of the research product: typically the publication or issued date. In case of a research result with different versions with different dates, the date of the result is selected as the most frequent well-formatted date. If not available, then the most recent and complete date among those that are well-formatted. For statistics, the year is extracted and the result is counted only among the result of that year. Example: Pre-print date: 2019-02-03, Article date provided by repository: 2020-02, Article date provided by Crossref: 2020, OpenAIRE will set as date 2019-02-03, because it’s the most recent among the complete and well-formed dates. If then the repository updates the metadata and set a complete date (e.g. 2020-02-12), then this will be the new date for the result because it becomes the most recent most complete date. However, if OpenAIRE then collects the pre-print from another repository with date 2019-02-03, then this will be the “winning date” because it becomes the most frequent well-formatted date." + }, + "publisher" : { + "type" : "string", + "description" : "The name of the entity that holds, archives, publishes prints, distributes, releases, issues, or produces the resource." + }, + "size" : { + "type" : "string", + "description" : "Only for results with type 'dataset': the declared size of the dataset" + }, + "source" : { + "description" : "See definition of Dublin Core field dc:source", + "type" : "array", + "items" : { + "type" : "string", + "description" : "See definition of Dublin Core field dc:source" + } + }, + "subjects" : { + "description" : "Keywords associated to the result", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "Why this subject is associated to the result" + } ] + }, + "subject" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "OpenAIRE subject classification scheme (https://api.openaire.eu/vocabularies/dnet:subject_classification_typologies)." + }, + "value" : { + "type" : "string", + "description" : "The value for the subject in the selected scheme. When the scheme is 'keyword', it means that the subject is free-text (i.e. not a term from a controlled vocabulary)." + } + } + } + }, + "description" : "Keywords associated to the result" + } + }, + "subtitle" : { + "type" : "string", + "description" : "Explanatory or alternative name by which a scientific result is known." + }, + "tool" : { + "description" : "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product" + } + }, + "type" : { + "type" : "string", + "description" : "Type of the result: one of 'publication', 'dataset', 'software', 'other' (see also https://api.openaire.eu/vocabularies/dnet:result_typologies)" + }, + "version" : { + "type" : "string", + "description" : "Version of the result" + } + } +} diff --git a/dump-schema/src/main/resources/jsonschemas/datasource_schema.json b/dump-schema/src/main/resources/jsonschemas/datasource_schema.json index 9ef7d38..a0eccc1 100644 --- a/dump-schema/src/main/resources/jsonschemas/datasource_schema.json +++ b/dump-schema/src/main/resources/jsonschemas/datasource_schema.json @@ -1,180 +1,196 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "accessrights": { - "type": "string", - "description": "Type of access to the data source, as defined by re3data.org. Possible values: {open, restricted, closed}" + "$schema" : "http://json-schema.org/draft-07/schema#", + "type" : "object", + "properties" : { + "accessrights" : { + "type" : "string", + "description" : "Type of access to the data source, as defined by re3data.org. Possible values: {open, restricted, closed}" }, - "certificates": { - "type": "string", - "description": "The certificate, seal or standard the data source complies with. As defined by re3data.org." + "certificates" : { + "type" : "string", + "description" : "The certificate, seal or standard the data source complies with. As defined by re3data.org." }, - "citationguidelineurl": { - "type": "string", - "description": "The URL of the data source providing information on how to cite its items. As defined by re3data.org." + "citationguidelineurl" : { + "type" : "string", + "description" : "The URL of the data source providing information on how to cite its items. As defined by re3data.org." }, - "contenttypes": { - "description": "Types of content in the data source, as defined by OpenDOAR", - "type": "array", - "items": { - "type": "string", - "description": "Types of content in the data source, as defined by OpenDOAR" + "contenttypes" : { + "description" : "Types of content in the data source, as defined by OpenDOAR", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Types of content in the data source, as defined by OpenDOAR" } }, - "databaseaccessrestriction": { - "type": "string", - "description": "Access restrinctions to the data source, as defined by re3data.org. One of {feeRequired, registration, other}" + "databaseaccessrestriction" : { + "type" : "string", + "description" : "Access restrinctions to the data source, as defined by re3data.org. One of {feeRequired, registration, other}" }, - "datasourcetype": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The scheme used to express the value (i.e. pubsrepository::journal)" + "datasourcetype" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme used to express the value (i.e. pubsrepository::journal)" }, - "value": { - "type": "string", - "description": "The value expressed in the scheme (Journal)" + "value" : { + "type" : "string", + "description" : "The value expressed in the scheme (Journal)" } }, - "description": "The type of the datasource. See https://api.openaire.eu/vocabularies/dnet:datasource_typologies" + "description" : "The type of the datasource. See https://api.openaire.eu/vocabularies/dnet:datasource_typologies" }, - "datauploadrestriction": { - "type": "string", - "description": "Upload restrictions applied by the datasource, as defined by re3data.org. One of {feeRequired, registration, other}" + "datauploadrestriction" : { + "type" : "string", + "description" : "Upload restrictions applied by the datasource, as defined by re3data.org. One of {feeRequired, registration, other}" }, - "dateofvalidation": { - "type": "string", - "description": "The date of last validation against the OpenAIRE guidelines for the datasource records" + "dateofvalidation" : { + "type" : "string", + "description" : "The date of last validation against the OpenAIRE guidelines for the datasource records" }, - "description": {"type": "string"}, - "englishname": { - "type": "string", - "description": "The English name of the datasource" + "description" : { + "type" : "string" }, - "id": { - "type": "string", - "description": "The OpenAIRE id of the data source" + "englishname" : { + "type" : "string", + "description" : "The English name of the datasource" }, - "journal": { - "type": "object", - "properties": { - "conferencedate": {"type": "string"}, - "conferenceplace": {"type": "string"}, - "edition": { - "type": "string", - "description": "Edition of the journal or conference proceeding" + "id" : { + "type" : "string", + "description" : "The OpenAIRE id of the data source" + }, + "journal" : { + "type" : "object", + "properties" : { + "conferencedate" : { + "type" : "string" }, - "ep": { - "type": "string", - "description": "End page" + "conferenceplace" : { + "type" : "string" }, - "iss": { - "type": "string", - "description": "Journal issue number" + "edition" : { + "type" : "string", + "description" : "Edition of the journal or conference proceeding" }, - "issnLinking": {"type": "string"}, - "issnOnline": {"type": "string"}, - "issnPrinted": {"type": "string"}, - "name": { - "type": "string", - "description": "Name of the journal or conference" + "ep" : { + "type" : "string", + "description" : "End page" }, - "sp": { - "type": "string", - "description": "Start page" + "iss" : { + "type" : "string", + "description" : "Journal issue number" }, - "vol": { - "type": "string", - "description": "Volume" + "issnLinking" : { + "type" : "string" + }, + "issnOnline" : { + "type" : "string" + }, + "issnPrinted" : { + "type" : "string" + }, + "name" : { + "type" : "string", + "description" : "Name of the journal or conference" + }, + "sp" : { + "type" : "string", + "description" : "Start page" + }, + "vol" : { + "type" : "string", + "description" : "Volume" } }, - "description": "Information about the journal, if this data source is of type Journal." + "description" : "Information about the journal, if this data source is of type Journal." }, - "languages": { - "description": "The languages present in the data source's content, as defined by OpenDOAR.", - "type": "array", - "items": { - "type": "string", - "description": "The languages present in the data source's content, as defined by OpenDOAR." + "languages" : { + "description" : "The languages present in the data source's content, as defined by OpenDOAR.", + "type" : "array", + "items" : { + "type" : "string", + "description" : "The languages present in the data source's content, as defined by OpenDOAR." } }, - "logourl": {"type": "string"}, - "missionstatementurl": { - "type": "string", - "description": "The URL of a mission statement describing the designated community of the data source. As defined by re3data.org" + "logourl" : { + "type" : "string" }, - "officialname": { - "type": "string", - "description": "The official name of the datasource" + "missionstatementurl" : { + "type" : "string", + "description" : "The URL of a mission statement describing the designated community of the data source. As defined by re3data.org" }, - "openairecompatibility": { - "type": "string", - "description": "OpenAIRE guidelines the data source comply with. See also https://guidelines.openaire.eu." + "officialname" : { + "type" : "string", + "description" : "The official name of the datasource" }, - "originalId": { - "description": "Original identifiers for the datasource", - "type": "array", - "items": { - "type": "string", - "description": "Original identifiers for the datasource" + "openairecompatibility" : { + "type" : "string", + "description" : "OpenAIRE guidelines the data source comply with. See also https://guidelines.openaire.eu." + }, + "originalId" : { + "description" : "Original identifiers for the datasource", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Original identifiers for the datasource" } }, - "pid": { - "description": "Persistent identifiers of the datasource", - "type": "array", - "items": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The scheme used to express the value " + "pid" : { + "description" : "Persistent identifiers of the datasource", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme used to express the value " }, - "value": { - "type": "string", - "description": "The value expressed in the scheme " + "value" : { + "type" : "string", + "description" : "The value expressed in the scheme " } }, - "description": "Persistent identifiers of the datasource" + "description" : "Persistent identifiers of the datasource" } }, - "pidsystems": { - "type": "string", - "description": "The persistent identifier system that is used by the data source. As defined by re3data.org" + "pidsystems" : { + "type" : "string", + "description" : "The persistent identifier system that is used by the data source. As defined by re3data.org" }, - "policies": { - "description": "Policies of the data source, as defined in OpenDOAR.", - "type": "array", - "items": { - "type": "string", - "description": "Policies of the data source, as defined in OpenDOAR." + "policies" : { + "description" : "Policies of the data source, as defined in OpenDOAR.", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Policies of the data source, as defined in OpenDOAR." } }, - "releaseenddate": { - "type": "string", - "description": "Date when the data source went offline or stopped ingesting new research data. As defined by re3data.org" + "releaseenddate" : { + "type" : "string", + "description" : "Date when the data source went offline or stopped ingesting new research data. As defined by re3data.org" }, - "releasestartdate": { - "type": "string", - "description": "Releasing date of the data source, as defined by re3data.org" + "releasestartdate" : { + "type" : "string", + "description" : "Releasing date of the data source, as defined by re3data.org" }, - "subjects": { - "description": "List of subjects associated to the datasource", - "type": "array", - "items": { - "type": "string", - "description": "List of subjects associated to the datasource" + "subjects" : { + "description" : "List of subjects associated to the datasource", + "type" : "array", + "items" : { + "type" : "string", + "description" : "List of subjects associated to the datasource" } }, - "uploadrights": { - "type": "string", - "description": "Type of data upload. As defined by re3data.org: one of {open, restricted,closed}" + "uploadrights" : { + "type" : "string", + "description" : "Type of data upload. As defined by re3data.org: one of {open, restricted,closed}" }, - "versioning": { - "type": "boolean", - "description": "As defined by redata.org: 'yes' if the data source supports versioning, 'no' otherwise." + "versioning" : { + "type" : "boolean", + "description" : "As defined by redata.org: 'yes' if the data source supports versioning, 'no' otherwise." }, - "websiteurl": {"type": "string"} + "websiteurl" : { + "type" : "string" + } } -} \ No newline at end of file +} diff --git a/dump-schema/src/main/resources/jsonschemas/openaire-community-dump-schema.json b/dump-schema/src/main/resources/jsonschemas/openaire-community-dump-schema.json index 2ca49f8..a30fe6a 100644 --- a/dump-schema/src/main/resources/jsonschemas/openaire-community-dump-schema.json +++ b/dump-schema/src/main/resources/jsonschemas/openaire-community-dump-schema.json @@ -1,563 +1,621 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "definitions": { - "CfHbKeyValue": { - "type": "object", - "properties": { - "key": { - "type": "string", - "description": "the OpenAIRE identifier of the data source" + "$schema" : "http://json-schema.org/draft-07/schema#", + "definitions" : { + "CfHbKeyValue" : { + "type" : "object", + "properties" : { + "key" : { + "type" : "string", + "description" : "the OpenAIRE identifier of the data source" }, - "value": { - "type": "string", - "description": "the name of the data source" + "value" : { + "type" : "string", + "description" : "the name of the data source" } } }, - "Provenance": { - "type": "object", - "properties": { - "provenance": {"type": "string"}, - "trust": {"type": "string"} - } - }, - "ResultPid": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The scheme of the persistent identifier for the result (i.e. doi). If the pid is here it means the information for the pid has been collected from an authority for that pid type (i.e. Crossref/Datacite for doi). The set of authoritative pid is: doi when collected from Crossref or Datacite pmid when collected from EuroPubmed, arxiv when collected from arXiv, handle from the repositories" + "Provenance" : { + "type" : "object", + "properties" : { + "provenance" : { + "type" : "string" }, - "value": { - "type": "string", - "description": "The value expressed in the scheme (i.e. 10.1000/182)" + "trust" : { + "type" : "string" } } }, - "Score": { - "type": "object", - "properties": { - "clazz": {"type": "string"}, - "score": {"type": "string"} + "ResultPid" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme of the persistent identifier for the result (i.e. doi). If the pid is here it means the information for the pid has been collected from an authority for that pid type (i.e. Crossref/Datacite for doi). The set of authoritative pid is: doi when collected from Crossref or Datacite pmid when collected from EuroPubmed, arxiv when collected from arXiv, handle from the repositories" + }, + "value" : { + "type" : "string", + "description" : "The value expressed in the scheme (i.e. 10.1000/182)" + } } } }, - "type": "object", - "properties": { - "author": { - "type": "array", - "items": { - "type": "object", - "properties": { - "fullname": {"type": "string"}, - "name": {"type": "string"}, - "pid": { - "type": "object", - "properties": { - "id": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The author's pid scheme. OpenAIRE currently supports 'ORCID'" + "type" : "object", + "properties" : { + "author" : { + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "fullname" : { + "type" : "string" + }, + "name" : { + "type" : "string" + }, + "pid" : { + "type" : "object", + "properties" : { + "id" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The author's pid scheme. OpenAIRE currently supports 'ORCID'" }, - "value": { - "type": "string", - "description": "The author's pid value in that scheme (i.e. 0000-1111-2222-3333)" + "value" : { + "type" : "string", + "description" : "The author's pid value in that scheme (i.e. 0000-1111-2222-3333)" } } }, - "provenance": { - "allOf": [ - {"$ref": "#/definitions/Provenance"}, - {"description": "The reason why the pid was associated to the author"} - ] + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "The reason why the pid was associated to the author" + } ] } }, - "description": "The author's persistent identifiers" + "description" : "The author's persistent identifiers" }, - "rank": {"type": "integer"}, - "surname": {"type": "string"} + "rank" : { + "type" : "integer" + }, + "surname" : { + "type" : "string" + } } } }, - "bestaccessright": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" + "bestaccessright" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" }, - "label": { - "type": "string", - "description": "Label for the access mode" + "label" : { + "type" : "string", + "description" : "Label for the access mode" }, - "scheme": { - "type": "string", - "description": "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" + "scheme" : { + "type" : "string", + "description" : "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" } }, - "description": "The openest of the access rights of this result." + "description" : "The openest of the access rights of this result." }, - "codeRepositoryUrl": { - "type": "string", - "description": "Only for results with type 'software': the URL to the repository with the source code" + "codeRepositoryUrl" : { + "type" : "string", + "description" : "Only for results with type 'software': the URL to the repository with the source code" }, - "collectedfrom": { - "description": "Information about the sources from which the record has been collected", - "type": "array", - "items": { - "allOf": [ - {"$ref": "#/definitions/CfHbKeyValue"}, - {"description": "Information about the sources from which the record has been collected"} - ] + "collectedfrom" : { + "description" : "Information about the sources from which the record has been collected", + "type" : "array", + "items" : { + "allOf" : [ { + "$ref" : "#/definitions/CfHbKeyValue" + }, { + "description" : "Information about the sources from which the record has been collected" + } ] } }, - "contactgroup": { - "description": "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource", - "type": "array", - "items": { - "type": "string", - "description": "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource" + "contactgroup" : { + "description" : "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource" } }, - "contactperson": { - "description": "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource", - "type": "array", - "items": { - "type": "string", - "description": "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource" + "contactperson" : { + "description" : "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource" } }, - "container": { - "type": "object", - "properties": { - "conferencedate": {"type": "string"}, - "conferenceplace": {"type": "string"}, - "edition": { - "type": "string", - "description": "Edition of the journal or conference proceeding" + "container" : { + "type" : "object", + "properties" : { + "conferencedate" : { + "type" : "string" }, - "ep": { - "type": "string", - "description": "End page" + "conferenceplace" : { + "type" : "string" }, - "iss": { - "type": "string", - "description": "Journal issue number" + "edition" : { + "type" : "string", + "description" : "Edition of the journal or conference proceeding" }, - "issnLinking": {"type": "string"}, - "issnOnline": {"type": "string"}, - "issnPrinted": {"type": "string"}, - "name": { - "type": "string", - "description": "Name of the journal or conference" + "ep" : { + "type" : "string", + "description" : "End page" }, - "sp": { - "type": "string", - "description": "Start page" + "iss" : { + "type" : "string", + "description" : "Journal issue number" }, - "vol": { - "type": "string", - "description": "Volume" + "issnLinking" : { + "type" : "string" + }, + "issnOnline" : { + "type" : "string" + }, + "issnPrinted" : { + "type" : "string" + }, + "name" : { + "type" : "string", + "description" : "Name of the journal or conference" + }, + "sp" : { + "type" : "string", + "description" : "Start page" + }, + "vol" : { + "type" : "string", + "description" : "Volume" } }, - "description": "Container has information about the conference or journal where the result has been presented or published" + "description" : "Container has information about the conference or journal where the result has been presented or published" }, - "context": { - "description": "Reference to a relevant research infrastructure, initiative or community (RI/RC) among those collaborating with OpenAIRE. Please see https://connect.openaire.eu", - "type": "array", - "items": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "Code identifying the RI/RC" + "context" : { + "description" : "Reference to a relevant research infrastructure, initiative or community (RI/RC) among those collaborating with OpenAIRE. Please see https://connect.openaire.eu", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "Code identifying the RI/RC" }, - "label": { - "type": "string", - "description": "Label of the RI/RC" + "label" : { + "type" : "string", + "description" : "Label of the RI/RC" }, - "provenance": { - "description": "Why this result is associated to the RI/RC.", - "type": "array", - "items": { - "allOf": [ - {"$ref": "#/definitions/Provenance"}, - {"description": "Why this result is associated to the RI/RC."} - ] + "provenance" : { + "description" : "Why this result is associated to the RI/RC.", + "type" : "array", + "items" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "Why this result is associated to the RI/RC." + } ] } } }, - "description": "Reference to a relevant research infrastructure, initiative or community (RI/RC) among those collaborating with OpenAIRE. Please see https://connect.openaire.eu" + "description" : "Reference to a relevant research infrastructure, initiative or community (RI/RC) among those collaborating with OpenAIRE. Please see https://connect.openaire.eu" } }, - "contributor": { - "description": "Contributors for the result", - "type": "array", - "items": { - "type": "string", - "description": "Contributors for the result" + "contributor" : { + "description" : "Contributors for the result", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Contributors for the result" } }, - "country": { - "description": "The list of countries associated to this result", - "type": "array", - "items": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "ISO 3166-1 alpha-2 country code (i.e. IT)" + "country" : { + "description" : "The list of countries associated to this result", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "ISO 3166-1 alpha-2 country code (i.e. IT)" }, - "label": { - "type": "string", - "description": "The label for that code (i.e. Italy)" + "label" : { + "type" : "string", + "description" : "The label for that code (i.e. Italy)" }, - "provenance": { - "allOf": [ - {"$ref": "#/definitions/Provenance"}, - {"description": "Why this result is associated to the country."} - ] + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "Why this result is associated to the country." + } ] } }, - "description": "The list of countries associated to this result" + "description" : "The list of countries associated to this result" } }, - "coverage": { - "type": "array", - "items": {"type": "string"} - }, - "dateofcollection": { - "type": "string", - "description": "When OpenAIRE collected the record the last time" - }, - "description": { - "type": "array", - "items": {"type": "string"} - }, - "documentationUrl": { - "description": "Only for results with type 'software': URL to the software documentation", - "type": "array", - "items": { - "type": "string", - "description": "Only for results with type 'software': URL to the software documentation" + "coverage" : { + "type" : "array", + "items" : { + "type" : "string" } }, - "embargoenddate": { - "type": "string", - "description": "Date when the embargo ends and this result turns Open Access" + "dateofcollection" : { + "type" : "string", + "description" : "When OpenAIRE collected the record the last time" }, - "format": { - "type": "array", - "items": {"type": "string"} - }, - "geolocation": { - "description": "Geolocation information", - "type": "array", - "items": { - "type": "object", - "properties": { - "box": {"type": "string"}, - "place": {"type": "string"}, - "point": {"type": "string"} - }, - "description": "Geolocation information" + "description" : { + "type" : "array", + "items" : { + "type" : "string" } }, - "id": { - "type": "string", - "description": "The OpenAIRE identifiers for this result" + "documentationUrl" : { + "description" : "Only for results with type 'software': URL to the software documentation", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': URL to the software documentation" + } }, - "indicators": { - "type": "object", - "properties": { - "impactMeasures": { - "type": "object", - "properties": { - "impulse": {"$ref": "#/definitions/Score"}, - "influence": {"$ref": "#/definitions/Score"}, - "influence_alt": {"$ref": "#/definitions/Score"}, - "popularity": {"$ref": "#/definitions/Score"}, - "popularity_alt": {"$ref": "#/definitions/Score"} + "embargoenddate" : { + "type" : "string", + "description" : "Date when the embargo ends and this result turns Open Access" + }, + "format" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "geolocation" : { + "description" : "Geolocation information", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "box" : { + "type" : "string" }, - "description": "The impact measures (i.e. popularity)" - }, - "usageCounts": { - "type": "object", - "properties": { - "downloads": {"type": "string"}, - "views": {"type": "string"} + "place" : { + "type" : "string" }, - "description": "The usage counts (i.e. downloads)" - } - }, - "description": "Indicators computed for this result, for example UsageCount ones" + "point" : { + "type" : "string" + } + }, + "description" : "Geolocation information" + } }, - "instance": { - "description": "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version", - "type": "array", - "items": { - "type": "object", - "properties": { - "accessright": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" + "id" : { + "type" : "string", + "description" : "The OpenAIRE identifiers for this result" + }, + "indicators" : { + "type" : "object", + "properties" : { + "bipIndicators" : { + "description" : "The impact measures (i.e. popularity)", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "clazz" : { + "type" : "string" }, - "label": { - "type": "string", - "description": "Label for the access mode" + "indicator" : { + "type" : "string" }, - "openAccessRoute": { - "type": "string", - "enum": [ - "gold", - "green", - "hybrid", - "bronze" - ] - }, - "scheme": { - "type": "string", - "description": "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" + "score" : { + "type" : "string" } }, - "description": "The accessRights for this materialization of the result" + "description" : "The impact measures (i.e. popularity)" + } + }, + "usageCounts" : { + "type" : "object", + "properties" : { + "downloads" : { + "type" : "string" + }, + "views" : { + "type" : "string" + } }, - "alternateIdentifier": { - "description": "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs", - "type": "array", - "items": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The scheme of the identifier. It can be a persistent identifier (i.e. doi). If it is present in the alternate identifiers it means it has not been forged by an authority for that pid. For example we collect metadata from an institutional repository that provides as identifier for the result also the doi" + "description" : "The usage counts (i.e. downloads)" + } + }, + "description" : "Indicators computed for this result, for example UsageCount ones" + }, + "instance" : { + "description" : "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "accessright" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" + }, + "label" : { + "type" : "string", + "description" : "Label for the access mode" + }, + "openAccessRoute" : { + "type" : "string", + "enum" : [ "gold", "green", "hybrid", "bronze" ] + }, + "scheme" : { + "type" : "string", + "description" : "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" + } + }, + "description" : "The accessRights for this materialization of the result" + }, + "alternateIdentifier" : { + "description" : "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme of the identifier. It can be a persistent identifier (i.e. doi). If it is present in the alternate identifiers it means it has not been forged by an authority for that pid. For example we collect metadata from an institutional repository that provides as identifier for the result also the doi" }, - "value": { - "type": "string", - "description": "The value expressed in the scheme" + "value" : { + "type" : "string", + "description" : "The value expressed in the scheme" } }, - "description": "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs" + "description" : "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs" } }, - "articleprocessingcharge": { - "type": "object", - "properties": { - "amount": {"type": "string"}, - "currency": {"type": "string"} + "articleprocessingcharge" : { + "type" : "object", + "properties" : { + "amount" : { + "type" : "string" + }, + "currency" : { + "type" : "string" + } }, - "description": "The money spent to make this book or article available in Open Access. Source for this information is the OpenAPC initiative." + "description" : "The money spent to make this book or article available in Open Access. Source for this information is the OpenAPC initiative." }, - "collectedfrom": { - "allOf": [ - {"$ref": "#/definitions/CfHbKeyValue"}, - {"description": "Information about the source from which the record has been collected"} - ] + "collectedfrom" : { + "allOf" : [ { + "$ref" : "#/definitions/CfHbKeyValue" + }, { + "description" : "Information about the source from which the record has been collected" + } ] }, - "hostedby": { - "allOf": [ - {"$ref": "#/definitions/CfHbKeyValue"}, - {"description": "Information about the source from which the instance can be viewed or downloaded."} - ] + "hostedby" : { + "allOf" : [ { + "$ref" : "#/definitions/CfHbKeyValue" + }, { + "description" : "Information about the source from which the instance can be viewed or downloaded." + } ] }, - "license": {"type": "string"}, - "pid": { - "type": "array", - "items": {"$ref": "#/definitions/ResultPid"} + "license" : { + "type" : "string" }, - "publicationdate": { - "type": "string", - "description": "Date of the research product" + "pid" : { + "type" : "array", + "items" : { + "$ref" : "#/definitions/ResultPid" + } }, - "refereed": { - "type": "string", - "description": "If this instance has been peer-reviewed or not. Allowed values are peerReviewed, nonPeerReviewed, UNKNOWN (as defined in https://api.openaire.eu/vocabularies/dnet:review_levels)" + "publicationdate" : { + "type" : "string", + "description" : "Date of the research product" }, - "type": { - "type": "string", - "description": "The specific sub-type of this instance (see https://api.openaire.eu/vocabularies/dnet:result_typologies following the links)" + "refereed" : { + "type" : "string", + "description" : "If this instance has been peer-reviewed or not. Allowed values are peerReviewed, nonPeerReviewed, UNKNOWN (as defined in https://api.openaire.eu/vocabularies/dnet:review_levels)" }, - "url": { - "description": "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. ", - "type": "array", - "items": { - "type": "string", - "description": "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. " + "type" : { + "type" : "string", + "description" : "The specific sub-type of this instance (see https://api.openaire.eu/vocabularies/dnet:result_typologies following the links)" + }, + "url" : { + "description" : "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. ", + "type" : "array", + "items" : { + "type" : "string", + "description" : "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. " } } }, - "description": "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version" + "description" : "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version" } }, - "language": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "alpha-3/ISO 639-2 code of the language" + "language" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "alpha-3/ISO 639-2 code of the language" }, - "label": { - "type": "string", - "description": "Language label in English" + "label" : { + "type" : "string", + "description" : "Language label in English" } } }, - "lastupdatetimestamp": { - "type": "integer", - "description": "Timestamp of last update of the record in OpenAIRE" + "lastupdatetimestamp" : { + "type" : "integer", + "description" : "Timestamp of last update of the record in OpenAIRE" }, - "maintitle": { - "type": "string", - "description": "A name or title by which a scientific result is known. May be the title of a publication, of a dataset or the name of a piece of software." + "maintitle" : { + "type" : "string", + "description" : "A name or title by which a scientific result is known. May be the title of a publication, of a dataset or the name of a piece of software." }, - "originalId": { - "description": "Identifiers of the record at the original sources", - "type": "array", - "items": { - "type": "string", - "description": "Identifiers of the record at the original sources" + "originalId" : { + "description" : "Identifiers of the record at the original sources", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Identifiers of the record at the original sources" } }, - "pid": { - "description": "Persistent identifiers of the result", - "type": "array", - "items": { - "allOf": [ - {"$ref": "#/definitions/ResultPid"}, - {"description": "Persistent identifiers of the result"} - ] + "pid" : { + "description" : "Persistent identifiers of the result", + "type" : "array", + "items" : { + "allOf" : [ { + "$ref" : "#/definitions/ResultPid" + }, { + "description" : "Persistent identifiers of the result" + } ] } }, - "programmingLanguage": { - "type": "string", - "description": "Only for results with type 'software': the programming language" + "programmingLanguage" : { + "type" : "string", + "description" : "Only for results with type 'software': the programming language" }, - "projects": { - "description": "List of projects (i.e. grants) that (co-)funded the production ofn the research results", - "type": "array", - "items": { - "type": "object", - "properties": { - "acronym": { - "type": "string", - "description": "The acronym of the project" + "projects" : { + "description" : "List of projects (i.e. grants) that (co-)funded the production ofn the research results", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "acronym" : { + "type" : "string", + "description" : "The acronym of the project" }, - "code": { - "type": "string", - "description": "The grant agreement number" + "code" : { + "type" : "string", + "description" : "The grant agreement number" }, - "funder": { - "type": "object", - "properties": { - "fundingStream": { - "type": "string", - "description": "Stream of funding (e.g. for European Commission can be H2020 or FP7)" + "funder" : { + "type" : "object", + "properties" : { + "fundingStream" : { + "type" : "string", + "description" : "Stream of funding (e.g. for European Commission can be H2020 or FP7)" }, - "jurisdiction": { - "type": "string", - "description": "Geographical jurisdiction (e.g. for European Commission is EU, for Croatian Science Foundation is HR)" + "jurisdiction" : { + "type" : "string", + "description" : "Geographical jurisdiction (e.g. for European Commission is EU, for Croatian Science Foundation is HR)" }, - "name": { - "type": "string", - "description": "The name of the funder (European Commission)" + "name" : { + "type" : "string", + "description" : "The name of the funder (European Commission)" }, - "shortName": { - "type": "string", - "description": "The short name of the funder (EC)" + "shortName" : { + "type" : "string", + "description" : "The short name of the funder (EC)" } }, - "description": "Information about the funder funding the project" + "description" : "Information about the funder funding the project" }, - "id": { - "type": "string", - "description": "The OpenAIRE id for the project" + "id" : { + "type" : "string", + "description" : "The OpenAIRE id for the project" }, - "provenance": {"$ref": "#/definitions/Provenance"}, - "title": {"type": "string"}, - "validated": { - "type": "object", - "properties": { - "validatedByFunder": {"type": "boolean"}, - "validationDate": {"type": "string"} - } - } - }, - "description": "List of projects (i.e. grants) that (co-)funded the production ofn the research results" - } - }, - "publicationdate": { - "type": "string", - "description": "Main date of the research product: typically the publication or issued date. In case of a research result with different versions with different dates, the date of the result is selected as the most frequent well-formatted date. If not available, then the most recent and complete date among those that are well-formatted. For statistics, the year is extracted and the result is counted only among the result of that year. Example: Pre-print date: 2019-02-03, Article date provided by repository: 2020-02, Article date provided by Crossref: 2020, OpenAIRE will set as date 2019-02-03, because it\u2019s the most recent among the complete and well-formed dates. If then the repository updates the metadata and set a complete date (e.g. 2020-02-12), then this will be the new date for the result because it becomes the most recent most complete date. However, if OpenAIRE then collects the pre-print from another repository with date 2019-02-03, then this will be the \u201cwinning date\u201d because it becomes the most frequent well-formatted date." - }, - "publisher": { - "type": "string", - "description": "The name of the entity that holds, archives, publishes prints, distributes, releases, issues, or produces the resource." - }, - "size": { - "type": "string", - "description": "Only for results with type 'dataset': the declared size of the dataset" - }, - "source": { - "description": "See definition of Dublin Core field dc:source", - "type": "array", - "items": { - "type": "string", - "description": "See definition of Dublin Core field dc:source" - } - }, - "subjects": { - "description": "Keywords associated to the result", - "type": "array", - "items": { - "type": "object", - "properties": { - "provenance": { - "allOf": [ - {"$ref": "#/definitions/Provenance"}, - {"description": "Why this subject is associated to the result"} - ] + "provenance" : { + "$ref" : "#/definitions/Provenance" }, - "subject": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "OpenAIRE subject classification scheme (https://api.openaire.eu/vocabularies/dnet:subject_classification_typologies)." + "title" : { + "type" : "string" + }, + "validated" : { + "type" : "object", + "properties" : { + "validatedByFunder" : { + "type" : "boolean" }, - "value": { - "type": "string", - "description": "The value for the subject in the selected scheme. When the scheme is 'keyword', it means that the subject is free-text (i.e. not a term from a controlled vocabulary)." + "validationDate" : { + "type" : "string" } } } }, - "description": "Keywords associated to the result" + "description" : "List of projects (i.e. grants) that (co-)funded the production ofn the research results" } }, - "subtitle": { - "type": "string", - "description": "Explanatory or alternative name by which a scientific result is known." + "publicationdate" : { + "type" : "string", + "description" : "Main date of the research product: typically the publication or issued date. In case of a research result with different versions with different dates, the date of the result is selected as the most frequent well-formatted date. If not available, then the most recent and complete date among those that are well-formatted. For statistics, the year is extracted and the result is counted only among the result of that year. Example: Pre-print date: 2019-02-03, Article date provided by repository: 2020-02, Article date provided by Crossref: 2020, OpenAIRE will set as date 2019-02-03, because it’s the most recent among the complete and well-formed dates. If then the repository updates the metadata and set a complete date (e.g. 2020-02-12), then this will be the new date for the result because it becomes the most recent most complete date. However, if OpenAIRE then collects the pre-print from another repository with date 2019-02-03, then this will be the “winning date” because it becomes the most frequent well-formatted date." }, - "tool": { - "description": "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product", - "type": "array", - "items": { - "type": "string", - "description": "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product" + "publisher" : { + "type" : "string", + "description" : "The name of the entity that holds, archives, publishes prints, distributes, releases, issues, or produces the resource." + }, + "size" : { + "type" : "string", + "description" : "Only for results with type 'dataset': the declared size of the dataset" + }, + "source" : { + "description" : "See definition of Dublin Core field dc:source", + "type" : "array", + "items" : { + "type" : "string", + "description" : "See definition of Dublin Core field dc:source" } }, - "type": { - "type": "string", - "description": "Type of the result: one of 'publication', 'dataset', 'software', 'other' (see also https://api.openaire.eu/vocabularies/dnet:result_typologies)" + "subjects" : { + "description" : "Keywords associated to the result", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "Why this subject is associated to the result" + } ] + }, + "subject" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "OpenAIRE subject classification scheme (https://api.openaire.eu/vocabularies/dnet:subject_classification_typologies)." + }, + "value" : { + "type" : "string", + "description" : "The value for the subject in the selected scheme. When the scheme is 'keyword', it means that the subject is free-text (i.e. not a term from a controlled vocabulary)." + } + } + } + }, + "description" : "Keywords associated to the result" + } }, - "version": { - "type": "string", - "description": "Version of the result" + "subtitle" : { + "type" : "string", + "description" : "Explanatory or alternative name by which a scientific result is known." + }, + "tool" : { + "description" : "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product" + } + }, + "type" : { + "type" : "string", + "description" : "Type of the result: one of 'publication', 'dataset', 'software', 'other' (see also https://api.openaire.eu/vocabularies/dnet:result_typologies)" + }, + "version" : { + "type" : "string", + "description" : "Version of the result" } } -} \ No newline at end of file +} diff --git a/dump-schema/src/main/resources/jsonschemas/organization_schema.json b/dump-schema/src/main/resources/jsonschemas/organization_schema.json index 6b2562b..43b4842 100644 --- a/dump-schema/src/main/resources/jsonschemas/organization_schema.json +++ b/dump-schema/src/main/resources/jsonschemas/organization_schema.json @@ -1,53 +1,59 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "alternativenames": { - "description": "Alternative names that identify the organisation", - "type": "array", - "items": { - "type": "string", - "description": "Alternative names that identify the organisation" + "$schema" : "http://json-schema.org/draft-07/schema#", + "type" : "object", + "properties" : { + "alternativenames" : { + "description" : "Alternative names that identify the organisation", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Alternative names that identify the organisation" } }, - "country": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "ISO 3166-1 alpha-2 country code (i.e. IT)" + "country" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "ISO 3166-1 alpha-2 country code (i.e. IT)" }, - "label": { - "type": "string", - "description": "The label for that code (i.e. Italy)" + "label" : { + "type" : "string", + "description" : "The label for that code (i.e. Italy)" } }, - "description": "The organisation country" + "description" : "The organisation country" }, - "id": { - "type": "string", - "description": "The OpenAIRE id for the organisation" + "id" : { + "type" : "string", + "description" : "The OpenAIRE id for the organisation" }, - "legalname": {"type": "string"}, - "legalshortname": {"type": "string"}, - "pid": { - "description": "Persistent identifiers for the organisation i.e. isni 0000000090326370", - "type": "array", - "items": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The scheme of the identifier (i.e. isni)" + "legalname" : { + "type" : "string" + }, + "legalshortname" : { + "type" : "string" + }, + "pid" : { + "description" : "Persistent identifiers for the organisation i.e. isni 0000000090326370", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme of the identifier (i.e. isni)" }, - "value": { - "type": "string", - "description": "The value in the schema (i.e. 0000000090326370)" + "value" : { + "type" : "string", + "description" : "The value in the schema (i.e. 0000000090326370)" } }, - "description": "Persistent identifiers for the organisation i.e. isni 0000000090326370" + "description" : "Persistent identifiers for the organisation i.e. isni 0000000090326370" } }, - "websiteurl": {"type": "string"} + "websiteurl" : { + "type" : "string" + } } -} \ No newline at end of file +} diff --git a/dump-schema/src/main/resources/jsonschemas/project_schema.json b/dump-schema/src/main/resources/jsonschemas/project_schema.json index c811872..b6dade7 100644 --- a/dump-schema/src/main/resources/jsonschemas/project_schema.json +++ b/dump-schema/src/main/resources/jsonschemas/project_schema.json @@ -1,119 +1,119 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "acronym": { - "type": "string" + "$schema" : "http://json-schema.org/draft-07/schema#", + "type" : "object", + "properties" : { + "acronym" : { + "type" : "string" }, - "callidentifier": { - "type": "string" + "callidentifier" : { + "type" : "string" }, - "code": { - "type": "string", - "description": "The grant agreement number" + "code" : { + "type" : "string" }, - "enddate": { - "type": "string" + "enddate" : { + "type" : "string" }, - "funding": { - "description": "Funding information for the project", - "type": "array", - "items": { - "type": "object", - "properties": { - "funding_stream": { - "type": "object", - "properties": { - "description": { - "type": "string", - "description": "Description of the funding stream" + "funding" : { + "description" : "Funding information for the project", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "funding_stream" : { + "type" : "object", + "properties" : { + "description" : { + "type" : "string" }, - "id": { - "type": "string", - "description": "Id of the funding stream" + "id" : { + "type" : "string", + "description" : "Id of the funding stream" } - } + }, + "description" : "Description of the funding stream" }, - "jurisdiction": { - "type": "string", - "description": "The jurisdiction of the funder (i.e. EU)" + "jurisdiction" : { + "type" : "string", + "description" : "Geographical jurisdiction (e.g. for European Commission is EU, for Croatian Science Foundation is HR)" }, - "name": { - "type": "string", - "description": "The name of the funder (European Commission)" + "name" : { + "type" : "string", + "description" : "The name of the funder (European Commission)" }, - "shortName": { - "type": "string", - "description": "The short name of the funder (EC)" + "shortName" : { + "type" : "string", + "description" : "The short name of the funder (EC)" } - } + }, + "description" : "Funding information for the project" } }, - "granted": { - "type": "object", - "properties": { - "currency": { - "type": "string", - "description": "The currency of the granted amount (e.g. EUR)" + "granted" : { + "type" : "object", + "properties" : { + "currency" : { + "type" : "string", + "description" : "The currency of the granted amount (e.g. EUR)" }, - "fundedamount": { - "type": "number", - "description": "The funded amount" + "fundedamount" : { + "type" : "number", + "description" : "The funded amount" }, - "totalcost": { - "type": "number", - "description": "The total cost of the project" + "totalcost" : { + "type" : "number", + "description" : "The total cost of the project" } }, - "description": "The money granted to the project" + "description" : "The money granted to the project" }, - "h2020programme": { - "description": "The h2020 programme funding the project", - "type": "array", - "items": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "The code of the programme" + "h2020programme" : { + "description" : "The h2020 programme funding the project", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "The code of the programme" }, - "description": { - "type": "string", - "description": "The description of the programme" + "description" : { + "type" : "string", + "description" : "The description of the programme" } - } + }, + "description" : "The h2020 programme funding the project" } }, - "id": { - "type": "string", - "description": "OpenAIRE id for the project" + "id" : { + "type" : "string" }, - "keywords": { - "type": "string" + "keywords" : { + "type" : "string" }, - "openaccessmandatefordataset": { - "type": "boolean" + "openaccessmandatefordataset" : { + "type" : "boolean" }, - "openaccessmandateforpublications": { - "type": "boolean" + "openaccessmandateforpublications" : { + "type" : "boolean" }, - "startdate": { - "type": "string" + "startdate" : { + "type" : "string" }, - "subject": { - "type": "array", - "items": { - "type": "string" + "subject" : { + "type" : "array", + "items" : { + "type" : "string" } }, - "summary": { - "type": "string" + "summary" : { + "type" : "string" }, - "title": { - "type": "string" + "title" : { + "type" : "string" }, - "websiteurl": { - "type": "string" + "websiteurl" : { + "type" : "string" } } -} \ No newline at end of file +} diff --git a/dump-schema/src/main/resources/jsonschemas/relation_schema.json b/dump-schema/src/main/resources/jsonschemas/relation_schema.json index 98134a0..d53d955 100644 --- a/dump-schema/src/main/resources/jsonschemas/relation_schema.json +++ b/dump-schema/src/main/resources/jsonschemas/relation_schema.json @@ -1,68 +1,54 @@ { - "$schema":"http://json-schema.org/draft-07/schema#", - "definitions": { - "Node": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "The OpenAIRE id of the entity" + "$schema" : "http://json-schema.org/draft-07/schema#", + "type" : "object", + "properties" : { + "provenance" : { + "type" : "object", + "properties" : { + "provenance" : { + "type" : "string" }, - "type": { - "type": "string", - "description": "The type of the entity (i.e. organisation)" - } - } - } - }, - "type":"object", - "properties": { - "provenance": { - "type": "object", - "properties": { - "provenance": { - "type": "string", - "description": "The reason why OpenAIRE holds the relation " - }, - "trust": { - "type": "string", - "description": "The trust of the relation in the range of [0,1]. Where greater the number, more the trust. Harvested relationships have typically a high trust (0.9). The trust of inferred relationship is calculated by the inference algorithm that generated them, as described in https://graph.openaire.eu/about#architecture (Enrichment --> Mining)" - } - } - }, - "reltype": { - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The semantics of the relation (i.e. isAuthorInstitutionOf). " - }, - "type": { - "type": "string", - "description": "the type of the relation (i.e. affiliation)" + "trust" : { + "type" : "string" } }, - "description": "To represent the semantics of a relation between two entities" + "description" : "The reason why OpenAIRE holds the relation " }, - "source": { - "allOf": [ - {"$ref": "#/definitions/Node"}, - {"description": "The node source in the relation"} - ] + "reltype" : { + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + }, + "type" : { + "type" : "string" + } + }, + "description" : "To represent the semantics of a relation between two entities" }, - "target": { - "allOf": [ - {"$ref": "#/definitions/Node"}, - {"description": "The node target in the relation"} - ] + "source" : { + "type" : "string", + "description" : "The identifier of the source in the relation" }, - "validated":{ - "type":"boolean", - "description":"True if the relation is related to a project and it has been collected from an authoritative source (i.e. the funder)" + "sourceType" : { + "type" : "string", + "description" : "The entity type of the source in the relation" }, - "validationDate":{ - "type":"string", - "description":"The date when the relation was collected from OpenAIRE" + "target" : { + "type" : "string", + "description" : "The identifier of the target in the relation" + }, + "targetType" : { + "type" : "string", + "description" : "The entity type of the target in the relation" + }, + "validated" : { + "type" : "boolean", + "description" : "True if the relation is related to a project and it has been collected from an authoritative source (i.e. the funder)" + }, + "validationDate" : { + "type" : "string", + "description" : "The date when the relation was collected from OpenAIRE" } } -} \ No newline at end of file +} diff --git a/dump-schema/src/main/resources/jsonschemas/result_schema.json b/dump-schema/src/main/resources/jsonschemas/result_schema.json index d9cc432..7120763 100644 --- a/dump-schema/src/main/resources/jsonschemas/result_schema.json +++ b/dump-schema/src/main/resources/jsonschemas/result_schema.json @@ -1,447 +1,493 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "definitions": { - "Provenance": { - "type": "object", - "properties": { - "provenance": {"type": "string"}, - "trust": {"type": "string"} - } - }, - "ResultPid": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The scheme of the persistent identifier for the result (i.e. doi). If the pid is here it means the information for the pid has been collected from an authority for that pid type (i.e. Crossref/Datacite for doi). The set of authoritative pid is: doi when collected from Crossref or Datacite pmid when collected from EuroPubmed, arxiv when collected from arXiv, handle from the repositories" + "$schema" : "http://json-schema.org/draft-07/schema#", + "definitions" : { + "Provenance" : { + "type" : "object", + "properties" : { + "provenance" : { + "type" : "string" }, - "value": { - "type": "string", - "description": "The value expressed in the scheme (i.e. 10.1000/182)" + "trust" : { + "type" : "string" } } }, - "Score": { - "type": "object", - "properties": { - "class": {"type": "string"}, - "score": {"type": "string"} + "ResultPid" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme of the persistent identifier for the result (i.e. doi). If the pid is here it means the information for the pid has been collected from an authority for that pid type (i.e. Crossref/Datacite for doi). The set of authoritative pid is: doi when collected from Crossref or Datacite pmid when collected from EuroPubmed, arxiv when collected from arXiv, handle from the repositories" + }, + "value" : { + "type" : "string", + "description" : "The value expressed in the scheme (i.e. 10.1000/182)" + } } } }, - "type": "object", - "properties": { - "author": { - "type": "array", - "items": { - "type": "object", - "properties": { - "fullname": {"type": "string"}, - "name": {"type": "string"}, - "pid": { - "type": "object", - "properties": { - "id": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The author's pid scheme. OpenAIRE currently supports 'ORCID'" + "type" : "object", + "properties" : { + "author" : { + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "fullname" : { + "type" : "string" + }, + "name" : { + "type" : "string" + }, + "pid" : { + "type" : "object", + "properties" : { + "id" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The author's pid scheme. OpenAIRE currently supports 'ORCID'" }, - "value": { - "type": "string", - "description": "The author's pid value in that scheme (i.e. 0000-1111-2222-3333)" + "value" : { + "type" : "string", + "description" : "The author's pid value in that scheme (i.e. 0000-1111-2222-3333)" } } }, - "provenance": { - "allOf": [ - {"$ref": "#/definitions/Provenance"}, - {"description": "The reason why the pid was associated to the author"} - ] + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "The reason why the pid was associated to the author" + } ] } }, - "description": "The author's persistent identifiers" + "description" : "The author's persistent identifiers" }, - "rank": {"type": "integer"}, - "surname": {"type": "string"} + "rank" : { + "type" : "integer" + }, + "surname" : { + "type" : "string" + } } } }, - "bestaccessright": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" + "bestaccessright" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" }, - "label": { - "type": "string", - "description": "Label for the access mode" + "label" : { + "type" : "string", + "description" : "Label for the access mode" }, - "scheme": { - "type": "string", - "description": "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" + "scheme" : { + "type" : "string", + "description" : "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" } }, - "description": "The openest of the access rights of this result." + "description" : "The openest of the access rights of this result." }, - "codeRepositoryUrl": { - "type": "string", - "description": "Only for results with type 'software': the URL to the repository with the source code" + "codeRepositoryUrl" : { + "type" : "string", + "description" : "Only for results with type 'software': the URL to the repository with the source code" }, - "contactgroup": { - "description": "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource", - "type": "array", - "items": { - "type": "string", - "description": "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource" + "contactgroup" : { + "description" : "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': Information on the group responsible for providing further information regarding the resource" } }, - "contactperson": { - "description": "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource", - "type": "array", - "items": { - "type": "string", - "description": "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource" + "contactperson" : { + "description" : "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': Information on the person responsible for providing further information regarding the resource" } }, - "container": { - "type": "object", - "properties": { - "conferencedate": {"type": "string"}, - "conferenceplace": {"type": "string"}, - "edition": { - "type": "string", - "description": "Edition of the journal or conference proceeding" + "container" : { + "type" : "object", + "properties" : { + "conferencedate" : { + "type" : "string" }, - "ep": { - "type": "string", - "description": "End page" + "conferenceplace" : { + "type" : "string" }, - "iss": { - "type": "string", - "description": "Journal issue number" + "edition" : { + "type" : "string", + "description" : "Edition of the journal or conference proceeding" }, - "issnLinking": {"type": "string"}, - "issnOnline": {"type": "string"}, - "issnPrinted": {"type": "string"}, - "name": { - "type": "string", - "description": "Name of the journal or conference" + "ep" : { + "type" : "string", + "description" : "End page" }, - "sp": { - "type": "string", - "description": "Start page" + "iss" : { + "type" : "string", + "description" : "Journal issue number" }, - "vol": { - "type": "string", - "description": "Volume" + "issnLinking" : { + "type" : "string" + }, + "issnOnline" : { + "type" : "string" + }, + "issnPrinted" : { + "type" : "string" + }, + "name" : { + "type" : "string", + "description" : "Name of the journal or conference" + }, + "sp" : { + "type" : "string", + "description" : "Start page" + }, + "vol" : { + "type" : "string", + "description" : "Volume" } }, - "description": "Container has information about the conference or journal where the result has been presented or published" + "description" : "Container has information about the conference or journal where the result has been presented or published" }, - "contributor": { - "description": "Contributors for the result", - "type": "array", - "items": { - "type": "string", - "description": "Contributors for the result" + "contributor" : { + "description" : "Contributors for the result", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Contributors for the result" } }, - "country": { - "description": "The list of countries associated to this result", - "type": "array", - "items": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "ISO 3166-1 alpha-2 country code (i.e. IT)" + "country" : { + "description" : "The list of countries associated to this result", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "ISO 3166-1 alpha-2 country code (i.e. IT)" }, - "label": { - "type": "string", - "description": "The label for that code (i.e. Italy)" + "label" : { + "type" : "string", + "description" : "The label for that code (i.e. Italy)" }, - "provenance": { - "allOf": [ - {"$ref": "#/definitions/Provenance"}, - {"description": "Why this result is associated to the country."} - ] + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "Why this result is associated to the country." + } ] } }, - "description": "The list of countries associated to this result" + "description" : "The list of countries associated to this result" } }, - "coverage": { - "type": "array", - "items": {"type": "string"} - }, - "dateofcollection": { - "type": "string", - "description": "When OpenAIRE collected the record the last time" - }, - "description": { - "type": "array", - "items": {"type": "string"} - }, - "documentationUrl": { - "description": "Only for results with type 'software': URL to the software documentation", - "type": "array", - "items": { - "type": "string", - "description": "Only for results with type 'software': URL to the software documentation" + "coverage" : { + "type" : "array", + "items" : { + "type" : "string" } }, - "embargoenddate": { - "type": "string", - "description": "Date when the embargo ends and this result turns Open Access" + "dateofcollection" : { + "type" : "string", + "description" : "When OpenAIRE collected the record the last time" }, - "format": { - "type": "array", - "items": {"type": "string"} - }, - "geolocation": { - "description": "Geolocation information", - "type": "array", - "items": { - "type": "object", - "properties": { - "box": {"type": "string"}, - "place": {"type": "string"}, - "point": {"type": "string"} - }, - "description": "Geolocation information" + "description" : { + "type" : "array", + "items" : { + "type" : "string" } }, - "id": { - "type": "string", - "description": "The OpenAIRE identifiers for this result" + "documentationUrl" : { + "description" : "Only for results with type 'software': URL to the software documentation", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'software': URL to the software documentation" + } }, - "indicators": { - "type": "object", - "properties": { - "impactMeasures": { - "type": "object", - "properties": { - "impulse": {"$ref": "#/definitions/Score"}, - "influence": {"$ref": "#/definitions/Score"}, - "influence_alt": {"$ref": "#/definitions/Score"}, - "popularity": {"$ref": "#/definitions/Score"}, - "popularity_alt": {"$ref": "#/definitions/Score"} + "embargoenddate" : { + "type" : "string", + "description" : "Date when the embargo ends and this result turns Open Access" + }, + "format" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "geolocation" : { + "description" : "Geolocation information", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "box" : { + "type" : "string" }, - "description": "The impact measures (i.e. popularity)" - }, - "usageCounts": { - "type": "object", - "properties": { - "downloads": {"type": "string"}, - "views": {"type": "string"} + "place" : { + "type" : "string" }, - "description": "The usage counts (i.e. downloads)" - } - }, - "description": "Indicators computed for this result, for example UsageCount ones" + "point" : { + "type" : "string" + } + }, + "description" : "Geolocation information" + } }, - "instance": { - "description": "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version", - "type": "array", - "items": { - "type": "object", - "properties": { - "accessright": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" + "id" : { + "type" : "string", + "description" : "The OpenAIRE identifiers for this result" + }, + "indicators" : { + "type" : "object", + "properties" : { + "bipIndicators" : { + "description" : "The impact measures (i.e. popularity)", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "class" : { + "type" : "string" }, - "label": { - "type": "string", - "description": "Label for the access mode" + "indicator" : { + "type" : "string" }, - "openAccessRoute": { - "type": "string", - "enum": [ - "gold", - "green", - "hybrid", - "bronze" - ] - }, - "scheme": { - "type": "string", - "description": "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" + "score" : { + "type" : "string" } }, - "description": "The accessRights for this materialization of the result" + "description" : "The impact measures (i.e. popularity)" + } + }, + "usageCounts" : { + "type" : "object", + "properties" : { + "downloads" : { + "type" : "string" + }, + "views" : { + "type" : "string" + } }, - "alternateIdentifier": { - "description": "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs", - "type": "array", - "items": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "The scheme of the identifier. It can be a persistent identifier (i.e. doi). If it is present in the alternate identifiers it means it has not been forged by an authority for that pid. For example we collect metadata from an institutional repository that provides as identifier for the result also the doi" + "description" : "The usage counts (i.e. downloads)" + } + }, + "description" : "Indicators computed for this result, for example UsageCount ones" + }, + "instance" : { + "description" : "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "accessright" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/" + }, + "label" : { + "type" : "string", + "description" : "Label for the access mode" + }, + "openAccessRoute" : { + "type" : "string", + "enum" : [ "gold", "green", "hybrid", "bronze" ] + }, + "scheme" : { + "type" : "string", + "description" : "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/" + } + }, + "description" : "The accessRights for this materialization of the result" + }, + "alternateIdentifier" : { + "description" : "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "The scheme of the identifier. It can be a persistent identifier (i.e. doi). If it is present in the alternate identifiers it means it has not been forged by an authority for that pid. For example we collect metadata from an institutional repository that provides as identifier for the result also the doi" }, - "value": { - "type": "string", - "description": "The value expressed in the scheme" + "value" : { + "type" : "string", + "description" : "The value expressed in the scheme" } }, - "description": "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs" + "description" : "All the identifiers other than pids forged by an authorithy for the pid type (i.e. Crossref for DOIs" } }, - "articleprocessingcharge": { - "type": "object", - "properties": { - "amount": {"type": "string"}, - "currency": {"type": "string"} + "articleprocessingcharge" : { + "type" : "object", + "properties" : { + "amount" : { + "type" : "string" + }, + "currency" : { + "type" : "string" + } }, - "description": "The money spent to make this book or article available in Open Access. Source for this information is the OpenAPC initiative." + "description" : "The money spent to make this book or article available in Open Access. Source for this information is the OpenAPC initiative." }, - "license": {"type": "string"}, - "pid": { - "type": "array", - "items": {"$ref": "#/definitions/ResultPid"} + "license" : { + "type" : "string" }, - "publicationdate": { - "type": "string", - "description": "Date of the research product" + "pid" : { + "type" : "array", + "items" : { + "$ref" : "#/definitions/ResultPid" + } }, - "refereed": { - "type": "string", - "description": "If this instance has been peer-reviewed or not. Allowed values are peerReviewed, nonPeerReviewed, UNKNOWN (as defined in https://api.openaire.eu/vocabularies/dnet:review_levels)" + "publicationdate" : { + "type" : "string", + "description" : "Date of the research product" }, - "type": { - "type": "string", - "description": "The specific sub-type of this instance (see https://api.openaire.eu/vocabularies/dnet:result_typologies following the links)" + "refereed" : { + "type" : "string", + "description" : "If this instance has been peer-reviewed or not. Allowed values are peerReviewed, nonPeerReviewed, UNKNOWN (as defined in https://api.openaire.eu/vocabularies/dnet:review_levels)" }, - "url": { - "description": "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. ", - "type": "array", - "items": { - "type": "string", - "description": "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. " + "type" : { + "type" : "string", + "description" : "The specific sub-type of this instance (see https://api.openaire.eu/vocabularies/dnet:result_typologies following the links)" + }, + "url" : { + "description" : "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. ", + "type" : "array", + "items" : { + "type" : "string", + "description" : "URLs to the instance. They may link to the actual full-text or to the landing page at the hosting source. " } } }, - "description": "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version" + "description" : "Each instance is one specific materialisation or version of the result. For example, you can have one result with three instance: one is the pre-print, one is the post-print, one is te published version" } }, - "language": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "alpha-3/ISO 639-2 code of the language" + "language" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "string", + "description" : "alpha-3/ISO 639-2 code of the language" }, - "label": { - "type": "string", - "description": "Language label in English" + "label" : { + "type" : "string", + "description" : "Language label in English" } } }, - "lastupdatetimestamp": { - "type": "integer", - "description": "Timestamp of last update of the record in OpenAIRE" + "lastupdatetimestamp" : { + "type" : "integer", + "description" : "Timestamp of last update of the record in OpenAIRE" }, - "maintitle": { - "type": "string", - "description": "A name or title by which a scientific result is known. May be the title of a publication, of a dataset or the name of a piece of software." + "maintitle" : { + "type" : "string", + "description" : "A name or title by which a scientific result is known. May be the title of a publication, of a dataset or the name of a piece of software." }, - "originalId": { - "description": "Identifiers of the record at the original sources", - "type": "array", - "items": { - "type": "string", - "description": "Identifiers of the record at the original sources" + "originalId" : { + "description" : "Identifiers of the record at the original sources", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Identifiers of the record at the original sources" } }, - "pid": { - "description": "Persistent identifiers of the result", - "type": "array", - "items": { - "allOf": [ - {"$ref": "#/definitions/ResultPid"}, - {"description": "Persistent identifiers of the result"} - ] + "pid" : { + "description" : "Persistent identifiers of the result", + "type" : "array", + "items" : { + "allOf" : [ { + "$ref" : "#/definitions/ResultPid" + }, { + "description" : "Persistent identifiers of the result" + } ] } }, - "programmingLanguage": { - "type": "string", - "description": "Only for results with type 'software': the programming language" + "programmingLanguage" : { + "type" : "string", + "description" : "Only for results with type 'software': the programming language" }, - "publicationdate": { - "type": "string", - "description": "Main date of the research product: typically the publication or issued date. In case of a research result with different versions with different dates, the date of the result is selected as the most frequent well-formatted date. If not available, then the most recent and complete date among those that are well-formatted. For statistics, the year is extracted and the result is counted only among the result of that year. Example: Pre-print date: 2019-02-03, Article date provided by repository: 2020-02, Article date provided by Crossref: 2020, OpenAIRE will set as date 2019-02-03, because it\u2019s the most recent among the complete and well-formed dates. If then the repository updates the metadata and set a complete date (e.g. 2020-02-12), then this will be the new date for the result because it becomes the most recent most complete date. However, if OpenAIRE then collects the pre-print from another repository with date 2019-02-03, then this will be the \u201cwinning date\u201d because it becomes the most frequent well-formatted date." + "publicationdate" : { + "type" : "string", + "description" : "Main date of the research product: typically the publication or issued date. In case of a research result with different versions with different dates, the date of the result is selected as the most frequent well-formatted date. If not available, then the most recent and complete date among those that are well-formatted. For statistics, the year is extracted and the result is counted only among the result of that year. Example: Pre-print date: 2019-02-03, Article date provided by repository: 2020-02, Article date provided by Crossref: 2020, OpenAIRE will set as date 2019-02-03, because it’s the most recent among the complete and well-formed dates. If then the repository updates the metadata and set a complete date (e.g. 2020-02-12), then this will be the new date for the result because it becomes the most recent most complete date. However, if OpenAIRE then collects the pre-print from another repository with date 2019-02-03, then this will be the “winning date” because it becomes the most frequent well-formatted date." }, - "publisher": { - "type": "string", - "description": "The name of the entity that holds, archives, publishes prints, distributes, releases, issues, or produces the resource." + "publisher" : { + "type" : "string", + "description" : "The name of the entity that holds, archives, publishes prints, distributes, releases, issues, or produces the resource." }, - "size": { - "type": "string", - "description": "Only for results with type 'dataset': the declared size of the dataset" + "size" : { + "type" : "string", + "description" : "Only for results with type 'dataset': the declared size of the dataset" }, - "source": { - "description": "See definition of Dublin Core field dc:source", - "type": "array", - "items": { - "type": "string", - "description": "See definition of Dublin Core field dc:source" + "source" : { + "description" : "See definition of Dublin Core field dc:source", + "type" : "array", + "items" : { + "type" : "string", + "description" : "See definition of Dublin Core field dc:source" } }, - "subjects": { - "description": "Keywords associated to the result", - "type": "array", - "items": { - "type": "object", - "properties": { - "provenance": { - "allOf": [ - {"$ref": "#/definitions/Provenance"}, - {"description": "Why this subject is associated to the result"} - ] + "subjects" : { + "description" : "Keywords associated to the result", + "type" : "array", + "items" : { + "type" : "object", + "properties" : { + "provenance" : { + "allOf" : [ { + "$ref" : "#/definitions/Provenance" + }, { + "description" : "Why this subject is associated to the result" + } ] }, - "subject": { - "type": "object", - "properties": { - "scheme": { - "type": "string", - "description": "OpenAIRE subject classification scheme (https://api.openaire.eu/vocabularies/dnet:subject_classification_typologies)." + "subject" : { + "type" : "object", + "properties" : { + "scheme" : { + "type" : "string", + "description" : "OpenAIRE subject classification scheme (https://api.openaire.eu/vocabularies/dnet:subject_classification_typologies)." }, - "value": { - "type": "string", - "description": "The value for the subject in the selected scheme. When the scheme is 'keyword', it means that the subject is free-text (i.e. not a term from a controlled vocabulary)." + "value" : { + "type" : "string", + "description" : "The value for the subject in the selected scheme. When the scheme is 'keyword', it means that the subject is free-text (i.e. not a term from a controlled vocabulary)." } } } }, - "description": "Keywords associated to the result" + "description" : "Keywords associated to the result" } }, - "subtitle": { - "type": "string", - "description": "Explanatory or alternative name by which a scientific result is known." + "subtitle" : { + "type" : "string", + "description" : "Explanatory or alternative name by which a scientific result is known." }, - "tool": { - "description": "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product", - "type": "array", - "items": { - "type": "string", - "description": "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product" + "tool" : { + "description" : "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product", + "type" : "array", + "items" : { + "type" : "string", + "description" : "Only for results with type 'other': tool useful for the interpretation and/or re-used of the research product" } }, - "type": { - "type": "string", - "description": "Type of the result: one of 'publication', 'dataset', 'software', 'other' (see also https://api.openaire.eu/vocabularies/dnet:result_typologies)" + "type" : { + "type" : "string", + "description" : "Type of the result: one of 'publication', 'dataset', 'software', 'other' (see also https://api.openaire.eu/vocabularies/dnet:result_typologies)" }, - "version": { - "type": "string", - "description": "Version of the result" + "version" : { + "type" : "string", + "description" : "Version of the result" } } -} \ No newline at end of file +} diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/CommunitySplit.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/CommunitySplit.java index 72af465..7fa3764 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/CommunitySplit.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/CommunitySplit.java @@ -55,34 +55,21 @@ public class CommunitySplit implements Serializable { communities .keySet() .stream() - .forEach(c -> printResult(c, result, outputPath + "/" + communities.get(c).replace(" ", "_"))); + .parallel() + .forEach(c -> { + result + .filter( + (FilterFunction) r -> Optional.ofNullable(r.getContext()).isPresent() && + r.getContext().stream().anyMatch(con -> con.getCode().equals(c))) + .map( + (MapFunction) cr -> new ObjectMapper().writeValueAsString(cr), + Encoders.STRING()) + .write() + .option("compression", "gzip") + .mode(SaveMode.Overwrite) + .text(outputPath + "/" + communities.get(c).replace(" ", "_")); + }); } - private static void printResult(String c, Dataset result, String outputPath) { - Dataset communityProducts = result - .filter((FilterFunction) r -> containsCommunity(r, c)); - - communityProducts - .map( - (MapFunction) cr -> new ObjectMapper().writeValueAsString(cr), - Encoders.STRING()) - .write() - .option("compression", "gzip") - .mode(SaveMode.Overwrite) - .text(outputPath); - - } - - private static boolean containsCommunity(CommunityResult r, String c) { - if (Optional.ofNullable(r.getContext()).isPresent()) { - return r - .getContext() - .stream() - .map(Context::getCode) - .collect(Collectors.toList()) - .contains(c); - } - return false; - } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java index 2d80d67..e9ad376 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java @@ -577,6 +577,10 @@ public class SparkDumpEntitiesJob implements Serializable { eu.dnetlib.dhp.schema.oaf.Organization org) { if (Boolean.TRUE.equals(org.getDataInfo().getDeletedbyinference())) return null; + if (!Optional.ofNullable(org.getLegalname()).isPresent() + && !Optional.ofNullable(org.getLegalshortname()).isPresent()) + return null; + Organization organization = new Organization(); Optional diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/country/SparkFindResultsRelatedToCountry.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/country/SparkFindResultsRelatedToCountry.java index b9531b8..0f4908c 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/country/SparkFindResultsRelatedToCountry.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/country/SparkFindResultsRelatedToCountry.java @@ -110,7 +110,9 @@ public class SparkFindResultsRelatedToCountry implements Serializable { Dataset organizationsInCountry = Utils .readPath(spark, inputPath + "/organization", Organization.class) - .filter((FilterFunction) o -> o.getCountry().getClassid().equals(country)); + .filter( + (FilterFunction) o -> !o.getDataInfo().getDeletedbyinference() + && o.getCountry().getClassid().equals(country)); Dataset relsOrganizationResults = Utils .readPath(spark, inputPath + "/relation", Relation.class) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/funderresults/SparkDumpFunderResults.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/funderresults/SparkDumpFunderResults.java index 60b8af1..668ae21 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/funderresults/SparkDumpFunderResults.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/funderresults/SparkDumpFunderResults.java @@ -11,6 +11,7 @@ import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.*; @@ -18,6 +19,8 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.model.community.CommunityResult; @@ -30,6 +33,7 @@ import eu.dnetlib.dhp.oa.model.community.Project; */ public class SparkDumpFunderResults implements Serializable { private static final Logger log = LoggerFactory.getLogger(SparkDumpFunderResults.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); public static void main(String[] args) throws Exception { String jsonConfiguration = IOUtils @@ -65,14 +69,24 @@ public class SparkDumpFunderResults implements Serializable { .union(Utils.readPath(spark, inputPath + "/otherresearchproduct", CommunityResult.class)) .union(Utils.readPath(spark, inputPath + "/software", CommunityResult.class)); log.info("Number of result {}", result.count()); + Dataset tmp = result .flatMap((FlatMapFunction) cr -> cr.getProjects().stream().map(p -> { return getFunderName(p); }).collect(Collectors.toList()).iterator(), Encoders.STRING()) .distinct(); List funderList = tmp.collectAsList(); - funderList.forEach(funder -> { - dumpResults(funder, result, outputPath); + funderList.stream().parallel().forEach(funder -> { + result + .filter( + (FilterFunction) r -> Optional.ofNullable(r.getProjects()).isPresent() && + r.getProjects().stream().anyMatch(p -> getFunderName(p).equals(funder))) + .map((MapFunction) r -> MAPPER.writeValueAsString(r), Encoders.STRING()) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .text(outputPath + "/" + funder); + }); } @@ -86,7 +100,7 @@ public class SparkDumpFunderResults implements Serializable { } return fName; } else { - String fName = p.getId().substring(3, p.getId().indexOf("_")).toUpperCase(); + String fName = p.getId().substring(0, p.getId().indexOf("_")).toUpperCase(); if (fName.equalsIgnoreCase("ec")) { if (p.getId().contains("he")) { fName += "_HE"; @@ -108,23 +122,4 @@ public class SparkDumpFunderResults implements Serializable { } } - private static void dumpResults(String funder, Dataset results, String outputPath) { - results.map((MapFunction) r -> { - if (!Optional.ofNullable(r.getProjects()).isPresent()) { - return null; - } - for (Project p : r.getProjects()) { - String fName = getFunderName(p); - if (fName.equalsIgnoreCase(funder)) { - return r; - } - } - return null; - }, Encoders.bean(CommunityResult.class)) - .filter(Objects::nonNull) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath + "/" + funder); - } } diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/countryresults/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/countryresults/oozie_app/workflow.xml index 19ba128..5d62bd9 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/countryresults/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/countryresults/oozie_app/workflow.xml @@ -80,8 +80,8 @@ - - + + @@ -239,34 +239,6 @@ - - - yarn - cluster - Select valid table relation - eu.dnetlib.dhp.oa.graph.dump.subset.SparkSelectSubset - dump-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - --conf spark.sql.shuffle.partitions=3840 - - --sourcePath${sourcePath} - --outputPath${outputPath} - --removeSet${removeSet} - - - - - - - diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml index d755629..e23778b 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml @@ -124,7 +124,7 @@ eu.dnetlib.dhp.oa.graph.dump.funderresults.SparkResultLinkedToProject dump-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} + --executor-memory=9G --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} @@ -132,6 +132,7 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=3840 --sourcePath${sourcePath}/publication --resultTableNameeu.dnetlib.dhp.schema.oaf.Publication diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml index d059346..ec18aaf 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml @@ -649,6 +649,7 @@ --hdfsPath${workingDir}/dump/relation/context --nameNode${nameNode} --isLookUpUrl${isLookUpUrl} + --masterDuplicate${workingDir}/masterduplicate diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/SplitForCommunityTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/SplitForCommunityTest.java index a24d9bf..1d48eda 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/SplitForCommunityTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/SplitForCommunityTest.java @@ -82,6 +82,7 @@ public class SplitForCommunityTest { .textFile(workingDir.toString() + "/split/Digital_Humanities_and_Cultural_Heritage") .map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class)); + System.out.println(tmp.count()); } @Test diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/communityResult/publication b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/communityResult/publication index b7f9ced..5e47fa8 100644 --- a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/communityResult/publication +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/communityResult/publication @@ -1 +1 @@ -{"pid": [{"scheme": "doi", "value": "10.1023/a:1019971625315"}], "contributor": [], "collectedfrom": [{"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}, {"key": "10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a", "value": "Microsoft Academic Graph"}], "id": "50|doi_________::0027accd79214af151336e8237a2b084", "container": {"issnPrinted": "1607-6729", "conferencedate": null, "vol": "385", "conferenceplace": null, "name": "Doklady Biochemistry and Biophysics", "iss": null, "sp": "228", "edition": null, "issnOnline": null, "ep": "234", "issnLinking": null}, "lastupdatetimestamp": 1649039791345, "author": [{"surname": null, "fullname": "Vladimir S. Saakov", "pid": null, "name": null, "rank": 1}], "instance": [{"refereed": "UNKNOWN", "hostedby": {"key": "10|issn___print::55156520c3996f4d887f858c089d1e5f", "value": "Doklady Biochemistry and Biophysics"}, "url": ["https://doi.org/10.1023/a:1019971625315"], "pid": [{"scheme": "doi", "value": "10.1023/a:1019971625315"}], "publicationdate": "2002-01-01", "collectedfrom": {"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}, "type": "Article"}], "subjects": [{"provenance": null, "subject": {"scheme": "keyword", "value": "General Chemistry"}}, {"provenance": null, "subject": {"scheme": "keyword", "value": "Biochemistry"}}, {"provenance": null, "subject": {"scheme": "keyword", "value": "General Medicine"}}, {"provenance": null, "subject": {"scheme": "keyword", "value": "Biophysics"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Photosystem II"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Ion"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Chemistry"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Soil salinity"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Analytical chemistry"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Function (biology)"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Pulse (signal processing)"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Fluorescence"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Phototroph"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Kinetic energy"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Photochemistry"}}], "publicationdate": "2002-01-01", "indicators": {"impactMeasures": {"influence": {"score": "4.901964E-9", "class": "C"}, "popularity": {"score": "6.185583E-10", "class": "C"}, "influence_alt": {"score": "3", "class": "C"}, "impulse": {"score": "0", "class": "C"}, "popularity_alt": {"score": "0.03722029", "class": "C"}}}, "dateofcollection": "2022-04-04T02:36:31Z", "type": "publication", "description": [], "format": [], "coverage": [], "publisher": "Springer Science and Business Media LLC", "language": {"code": "und", "label": "Undetermined"}, "country": [], "originalId": ["453197", "10.1023/a:1019971625315", "314096869"], "source": ["Crossref", null], "context": [{"code": "enermaps", "provenance": [{"provenance": "Inferred by OpenAIRE", "trust": "0.8"}], "label": "Energy Research"}]} \ No newline at end of file +{"pid": [{"scheme": "doi", "value": "10.1023/a:1019971625315"}], "contributor": [], "collectedfrom": [{"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}, {"key": "10|openaire____::5f532a3fc4f1ea403f37070f59a7a53a", "value": "Microsoft Academic Graph"}], "id": "50|doi_________::0027accd79214af151336e8237a2b084", "container": {"issnPrinted": "1607-6729", "conferencedate": null, "vol": "385", "conferenceplace": null, "name": "Doklady Biochemistry and Biophysics", "iss": null, "sp": "228", "edition": null, "issnOnline": null, "ep": "234", "issnLinking": null}, "lastupdatetimestamp": 1649039791345, "author": [{"surname": null, "fullname": "Vladimir S. Saakov", "pid": null, "name": null, "rank": 1}], "instance": [{"refereed": "UNKNOWN", "hostedby": {"key": "10|issn___print::55156520c3996f4d887f858c089d1e5f", "value": "Doklady Biochemistry and Biophysics"}, "url": ["https://doi.org/10.1023/a:1019971625315"], "pid": [{"scheme": "doi", "value": "10.1023/a:1019971625315"}], "publicationdate": "2002-01-01", "collectedfrom": {"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}, "type": "Article"}], "subjects": [{"provenance": null, "subject": {"scheme": "keyword", "value": "General Chemistry"}}, {"provenance": null, "subject": {"scheme": "keyword", "value": "Biochemistry"}}, {"provenance": null, "subject": {"scheme": "keyword", "value": "General Medicine"}}, {"provenance": null, "subject": {"scheme": "keyword", "value": "Biophysics"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Photosystem II"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Ion"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Chemistry"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Soil salinity"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Analytical chemistry"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Function (biology)"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Pulse (signal processing)"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Fluorescence"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Phototroph"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Kinetic energy"}}, {"provenance": null, "subject": {"scheme": "MAG", "value": "Photochemistry"}}], "publicationdate": "2002-01-01", "indicators": {"bipIndicators": [{"indicator":"influence","score": "4.901964E-9", "class": "C"}, {"indicator":"popularity","score": "6.185583E-10", "class": "C"}, {"indicator": "influence_alt","score": "3", "class": "C"}, {"indicator": "impulse","score": "0", "class": "C"}, {"indicator": "popularity_alt","score": "0.03722029", "class": "C"}]}}, "dateofcollection": "2022-04-04T02:36:31Z", "type": "publication", "description": [], "format": [], "coverage": [], "publisher": "Springer Science and Business Media LLC", "language": {"code": "und", "label": "Undetermined"}, "country": [], "originalId": ["453197", "10.1023/a:1019971625315", "314096869"], "source": ["Crossref", null], "context": [{"code": "enermaps", "provenance": [{"provenance": "Inferred by OpenAIRE", "trust": "0.8"}], "label": "Energy Research"}]} \ No newline at end of file