From db36a9be2e78b011e577f58f4841ba67890a9541 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 22 Dec 2022 09:38:09 +0100 Subject: [PATCH] [Dump Subset] issue on the relations --- .../dhp/oa/graph/dump/ResultMapper.java | 2 +- .../dump/complete/QueryInformationSystem.java | 6 +- .../oa/graph/dump/subset/SparkDumpResult.java | 19 ++- .../graph/dump/subset/SparkSelectSubset.java | 61 +++---- .../dump/subset/SparkSelectValidContext.java | 2 +- .../dump/subset/SparkSelectValidRelation.java | 151 ++++++++++++++++++ ...nput_select_valid_relation_parameters.json | 27 ++++ .../subset/oozie_app/workflow.xml | 43 +++-- .../dhp/oa/graph/dump/DumpJobTest.java | 31 ++++ ...DumpOrganizationProjectDatasourceTest.java | 33 ++++ .../dump/funderresult/SplitPerFunderTest.java | 2 + .../datasourcenotdumped/datasource.json | 1 + .../dump/resultDump/resultNotDumped.json | 1 + 13 files changed, 330 insertions(+), 49 deletions(-) create mode 100644 dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelation.java create mode 100644 dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_select_valid_relation_parameters.json create mode 100644 dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/datasourcenotdumped/datasource.json create mode 100644 dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/resultDump/resultNotDumped.json diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java index 1526fad..f26a4b4 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java @@ -323,7 +323,7 @@ public class ResultMapper implements Serializable { } } } catch (ClassCastException cce) { - return out; + return null; } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java index b681fd3..bbcb1a2 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java @@ -14,15 +14,18 @@ import org.dom4j.Element; import org.dom4j.Node; import org.dom4j.io.SAXReader; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; +import eu.dnetlib.dhp.oa.graph.dump.subset.SparkDumpResult; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.utils.DHPUtils; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; public class QueryInformationSystem { - + private static final Logger log = LoggerFactory.getLogger(QueryInformationSystem.class); private ISLookUpService isLookUp; private List contextRelationResult; @@ -51,6 +54,7 @@ public class QueryInformationSystem { String[] cSplit = c.split("@@"); cinfo.setId(cSplit[0]); cinfo.setName(cSplit[1]); + log.info("community name : {}", cSplit[1]); cinfo.setDescription(cSplit[2]); if (!cSplit[3].trim().equals("")) { cinfo.setSubject(Arrays.asList(cSplit[3].split(","))); diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkDumpResult.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkDumpResult.java index 4698be9..9b082ab 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkDumpResult.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkDumpResult.java @@ -34,15 +34,11 @@ import eu.dnetlib.dhp.oa.graph.dump.subset.criteria.VerbResolver; import eu.dnetlib.dhp.oa.graph.dump.subset.criteria.VerbResolverFactory; import eu.dnetlib.dhp.oa.graph.dump.subset.selectionconstraints.Param; import eu.dnetlib.dhp.oa.graph.dump.subset.selectionconstraints.SelectionConstraints; -import eu.dnetlib.dhp.oa.model.Container; import eu.dnetlib.dhp.oa.model.graph.*; -import eu.dnetlib.dhp.oa.model.graph.Datasource; -import eu.dnetlib.dhp.oa.model.graph.Organization; -import eu.dnetlib.dhp.oa.model.graph.Project; import eu.dnetlib.dhp.schema.oaf.*; /** - * Spark Job that fires the dump for the entites + * Spark Job that fires the dump for the entities */ public class SparkDumpResult implements Serializable { private static final Logger log = LoggerFactory.getLogger(SparkDumpResult.class); @@ -142,7 +138,7 @@ public class SparkDumpResult implements Serializable { .readPath(spark, inputPath, inputClazz) .map( (MapFunction) value -> filterResult( - value, pathMap, selectionConstraints, inputClazz, masterDuplicateList), + value, pathMap, selectionConstraints, inputClazz, masterDuplicateList, resultType), Encoders.bean(inputClazz)) .filter(Objects::nonNull) .write() @@ -167,7 +163,8 @@ public class SparkDumpResult implements Serializable { } private static I filterResult(I value, Map pathMap, - SelectionConstraints selectionConstraints, Class inputClazz, List masterDuplicateList) { + SelectionConstraints selectionConstraints, Class inputClazz, List masterDuplicateList, + String resultType) { Optional odInfo = Optional.ofNullable(value.getDataInfo()); if (Boolean.FALSE.equals(odInfo.isPresent())) { @@ -178,6 +175,10 @@ public class SparkDumpResult implements Serializable { return null; } + if (!isCompatible(value.getResulttype().getClassid(), resultType)) { + return null; + } + if (selectionConstraints != null) { Param param = new Param(); String json = new Gson().toJson(value, inputClazz); @@ -205,6 +206,10 @@ public class SparkDumpResult implements Serializable { return value; } + private static boolean isCompatible(String classid, String resultType) { + return (classid.equals(resultType) || (classid.equals("other") && resultType.equals("otherresearchproduct"))); + } + private static void update(KeyValue kv, List masterDuplicateList) { for (MasterDuplicate md : masterDuplicateList) { if (md.getDuplicate().equals(kv.getKey())) { diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectSubset.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectSubset.java index 08acb10..5c02498 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectSubset.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectSubset.java @@ -85,32 +85,32 @@ public class SparkSelectSubset implements Serializable { && !removeSet.contains(r.getRelClass())); Dataset resultIds = Utils - .readPath(spark, outputPath + "/dump/publication", GraphResult.class) + .readPath(spark, outputPath + "/original/publication", Publication.class) - .map((MapFunction) p -> p.getId(), Encoders.STRING()) + .map((MapFunction) p -> p.getId(), Encoders.STRING()) .union( Utils - .readPath(spark, outputPath + "/dump/dataset", GraphResult.class) + .readPath(spark, outputPath + "/original/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class) - .map((MapFunction) d -> d.getId(), Encoders.STRING())) + .map((MapFunction) d -> d.getId(), Encoders.STRING())) .union( Utils - .readPath(spark, outputPath + "/dump/software", GraphResult.class) + .readPath(spark, outputPath + "/original/software", Software.class) - .map((MapFunction) s -> s.getId(), Encoders.STRING())) + .map((MapFunction) s -> s.getId(), Encoders.STRING())) .union( Utils - .readPath(spark, outputPath + "/dump/otherresearchproduct", GraphResult.class) + .readPath(spark, outputPath + "/original/otherresearchproduct", OtherResearchProduct.class) - .map((MapFunction) o -> o.getId(), Encoders.STRING())); + .map((MapFunction) o -> o.getId(), Encoders.STRING())); // select result -> result relations - Dataset relJoinSource = relation + Dataset relResultResult = relation .joinWith(resultIds, relation.col("source").equalTo(resultIds.col("value"))) .map((MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)); - relJoinSource - .joinWith(resultIds, relJoinSource.col("target").equalTo(resultIds.col("value"))) + relResultResult + .joinWith(resultIds, relResultResult.col("target").equalTo(resultIds.col("value"))) .map((MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)) .write() .option("compression", "gzip") @@ -133,40 +133,45 @@ public class SparkSelectSubset implements Serializable { .filter((FilterFunction) e -> !e.getDataInfo().getDeletedbyinference()) .map((MapFunction) d -> d.getId(), Encoders.STRING())); - relJoinSource = relation + Dataset relResultOther = relation .joinWith(resultIds, relation.col("source").equalTo(resultIds.col("value"))) .map((MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)); - relJoinSource - .joinWith(otherIds, relJoinSource.col("target").equalTo(otherIds.col("value"))) + relResultOther + .joinWith(otherIds, relResultOther.col("target").equalTo(otherIds.col("value"))) .map((MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)) .write() .mode(SaveMode.Append) .option("compression", "gzip") .json(outputPath + "/original/relation"); - relJoinSource = relation + Dataset relOtherResult = relation .joinWith(resultIds, relation.col("target").equalTo(resultIds.col("value"))) .map((MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)); - relJoinSource - .joinWith(otherIds, relJoinSource.col("source").equalTo(otherIds.col("value"))) + relOtherResult + .joinWith(otherIds, relOtherResult.col("source").equalTo(otherIds.col("value"))) .map((MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)) .write() .mode(SaveMode.Append) .option("compression", "gzip") .json(outputPath + "/original/relation"); - relJoinSource = Utils.readPath(spark, outputPath + "/original/relation", Relation.class); + Dataset relAll = Utils + .readPath(spark, outputPath + "/original/relation", Relation.class) + .flatMap( + (FlatMapFunction) r -> Arrays.asList(r.getSource(), r.getTarget()).iterator(), + Encoders.STRING()) + .distinct(); + // Save the entities in relations with at least one result - // relations are bidirectional, so it is enough to have the match to one side Dataset organization = Utils .readPath(spark, inputPath + "/organization", Organization.class) .filter((FilterFunction) o -> !o.getDataInfo().getDeletedbyinference()); organization - .joinWith(relJoinSource, organization.col("id").equalTo(relJoinSource.col("source"))) + .joinWith(relAll, organization.col("id").equalTo(relAll.col("value"))) .map( - (MapFunction, Organization>) t2 -> t2._1(), + (MapFunction, Organization>) t2 -> t2._1(), Encoders.bean(Organization.class)) .groupByKey((MapFunction) v -> v.getId(), Encoders.STRING()) .mapGroups( @@ -181,8 +186,8 @@ public class SparkSelectSubset implements Serializable { .readPath(spark, inputPath + "/datasource", Datasource.class) .filter((FilterFunction) d -> !d.getDataInfo().getDeletedbyinference()); datasource - .joinWith(relJoinSource, datasource.col("id").equalTo(relJoinSource.col("source"))) - .map((MapFunction, Datasource>) t2 -> t2._1(), Encoders.bean(Datasource.class)) + .joinWith(relAll, datasource.col("id").equalTo(relAll.col("value"))) + .map((MapFunction, Datasource>) t2 -> t2._1(), Encoders.bean(Datasource.class)) .groupByKey((MapFunction) v -> v.getId(), Encoders.STRING()) .mapGroups( (MapGroupsFunction) (k, it) -> it.next(), @@ -283,8 +288,8 @@ public class SparkSelectSubset implements Serializable { .readPath(spark, inputPath + "/project", Project.class) .filter((FilterFunction) d -> !d.getDataInfo().getDeletedbyinference()); project - .joinWith(relJoinSource, project.col("id").equalTo(relJoinSource.col("source"))) - .map((MapFunction, Project>) t2 -> t2._1(), Encoders.bean(Project.class)) + .joinWith(relAll, project.col("id").equalTo(relAll.col("value"))) + .map((MapFunction, Project>) t2 -> t2._1(), Encoders.bean(Project.class)) .groupByKey((MapFunction) v -> v.getId(), Encoders.STRING()) .mapGroups((MapGroupsFunction) (k, it) -> it.next(), Encoders.bean(Project.class)) .write() @@ -306,12 +311,12 @@ public class SparkSelectSubset implements Serializable { .readPath(spark, outputPath + "/original/datasource", Datasource.class) .map((MapFunction) d -> d.getId(), Encoders.STRING())); - relJoinSource = relation + Dataset relOtherOther = relation .joinWith(selectedIDs, relation.col("source").equalTo(selectedIDs.col("value"))) .map((MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)); - relJoinSource - .joinWith(selectedIDs, relJoinSource.col("target").equalTo(selectedIDs.col("value"))) + relOtherOther + .joinWith(selectedIDs, relOtherOther.col("target").equalTo(selectedIDs.col("value"))) .map((MapFunction, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class)) .write() .mode(SaveMode.Append) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidContext.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidContext.java index 9c8e049..0b90c70 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidContext.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidContext.java @@ -127,7 +127,7 @@ public class SparkSelectValidContext implements Serializable { private static boolean extracted(String c, List keySet) { if (keySet.contains(c)) return true; - if (c.contains("::") && keySet.contains(c.substring(0, c.indexOf("::")))) + if (c.contains(":") && keySet.contains(c.substring(0, c.indexOf(":")))) return true; return false; } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelation.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelation.java new file mode 100644 index 0000000..84ca4f6 --- /dev/null +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/subset/SparkSelectValidRelation.java @@ -0,0 +1,151 @@ + +package eu.dnetlib.dhp.oa.graph.dump.subset; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.Serializable; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.oa.graph.dump.Utils; +import eu.dnetlib.dhp.oa.model.graph.GraphResult; +import eu.dnetlib.dhp.oa.model.graph.Relation; +import eu.dnetlib.dhp.oa.model.graph.ResearchCommunity; +import scala.Tuple2; + +/** + * @author miriam.baglioni + * @Date 15/11/22 + */ +public class SparkSelectValidRelation implements Serializable { + private static final Logger log = LoggerFactory.getLogger(SparkSelectValidRelation.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + SparkSelectValidRelation.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/dump/input_select_valid_relation_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + // results dumped + final String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + // all relations plus those produced via context and extracted from results + final String relationPath = parser.get("relationPath"); + log.info("relationPath: {}", relationPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + selectValidRelation(spark, inputPath, relationPath); + + }); + + } + + private static void selectValidRelation(SparkSession spark, String inputPath, + String relationPath) { + // read the results + Dataset dumpedIds = Utils + .readPath(spark, inputPath + "/publication", GraphResult.class) + .map((MapFunction) r -> r.getId(), Encoders.STRING()) + .union( + Utils + .readPath(spark, inputPath + "/dataset", GraphResult.class) + .map((MapFunction) r -> r.getId(), Encoders.STRING())) + .union( + Utils + .readPath(spark, inputPath + "/software", GraphResult.class) + .map((MapFunction) r -> r.getId(), Encoders.STRING())) + .union( + Utils + .readPath(spark, inputPath + "/otherresearchproduct", GraphResult.class) + .map((MapFunction) r -> r.getId(), Encoders.STRING())) + .union( + Utils + .readPath(spark, inputPath + "/organization", eu.dnetlib.dhp.oa.model.graph.Organization.class) + .map( + (MapFunction) o -> o.getId(), + Encoders.STRING())) + .union( + Utils + .readPath(spark, inputPath + "/project", eu.dnetlib.dhp.oa.model.graph.Project.class) + .map( + (MapFunction) o -> o.getId(), Encoders.STRING())) + .union( + Utils + .readPath(spark, inputPath + "/datasource", eu.dnetlib.dhp.oa.model.graph.Datasource.class) + .map( + (MapFunction) o -> o.getId(), + Encoders.STRING())) + .union( + Utils + .readPath(spark, inputPath + "/community_infrastructure", ResearchCommunity.class) + .map((MapFunction) c -> c.getId(), Encoders.STRING())); + + Dataset> relationSource = Utils + .readPath(spark, relationPath, Relation.class) + .map( + (MapFunction>) r -> new Tuple2<>(r.getSource().getId(), r), + Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))); + + Dataset> relJoinSource = relationSource + .joinWith(dumpedIds, relationSource.col("_1").equalTo(dumpedIds.col("value"))) + .map( + (MapFunction, String>, Tuple2>) t2 -> new Tuple2<>( + t2._1()._2().getTarget().getId(), t2._1()._2()), + Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))); + + relJoinSource + .joinWith(dumpedIds, relJoinSource.col("_1").equalTo(dumpedIds.col("value"))) + .map( + (MapFunction, String>, Relation>) t2 -> t2._1()._2(), + Encoders.bean(Relation.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(inputPath + "/relation"); + +// relJoinSource = relationSource +// .joinWith(dumpedIds, relationSource.col("_1").equalTo(dumpedIds.col("value"))) +// .map( +// (MapFunction, String>, Tuple2>) t2 -> new Tuple2<>( +// t2._1()._2().getTarget().getId(), t2._1()._2()), +// Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))); +// +// relJoinSource +// .joinWith(dumpedIds, relJoinSource.col("_1").equalTo(dumpedIds.col("value"))) +// .map( +// (MapFunction, String>, Relation>) t2 -> t2._1()._2(), +// Encoders.bean(Relation.class)) +// .write() +// .mode(SaveMode.Append) +// .option("compression", "gzip") +// .json(inputPath + "/relation"); + + } + +} diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_select_valid_relation_parameters.json b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_select_valid_relation_parameters.json new file mode 100644 index 0000000..91b4d20 --- /dev/null +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_select_valid_relation_parameters.json @@ -0,0 +1,27 @@ +[ + + + { + "paramName":"s", + "paramLongName":"sourcePath", + "paramDescription": "the path of the sequencial file to read", + "paramRequired": true + }, + + { + "paramName": "ssm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "true if the spark session is managed, false otherwise", + "paramRequired": false + }, + + { + "paramName":"rp", + "paramLongName":"relationPath", + "paramDescription": "the map to find fields in the json", + "paramRequired": false + } +] + + + diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml index e19ea61..69117be 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml @@ -349,8 +349,6 @@ - - eu.dnetlib.dhp.oa.graph.dump.complete.CreateContextEntities @@ -390,7 +388,7 @@ eu.dnetlib.dhp.oa.graph.dump.complete.CreateContextRelation - --hdfsPath${workingDir}/relation/context + --hdfsPath${workingDir}/dump/relation/context --nameNode${nameNode} --isLookUpUrl${isLookUpUrl} @@ -416,7 +414,7 @@ --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --sourcePath${sourcePath}/relation - --outputPath${workingDir}/relation/contextOrg + --outputPath${workingDir}/dump/relation/contextOrg --organizationCommunityMap${organizationCommunityMap} --communityMapPath${communityMapPath} @@ -470,7 +468,7 @@ --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --sourcePath${outputPath}/original/relation - --outputPath${outputPath}/dump/relation + --outputPath${workingDir}/relation --removeSet${removeSet} @@ -496,7 +494,7 @@ --sourcePath${outputPath}/original/publication --resultTableNameeu.dnetlib.dhp.schema.oaf.Publication - --outputPath${outputPath}/dump/relation + --outputPath${workingDir}/relation --communityMapPath${communityMapPath} @@ -522,7 +520,7 @@ --sourcePath${outputPath}/original/dataset --resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset - --outputPath${outputPath}/dump/relation + --outputPath${workingDir}/relation --communityMapPath${communityMapPath} @@ -548,7 +546,7 @@ --sourcePath${outputPath}/original/otherresearchproduct --resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct - --outputPath${outputPath}/dump/relation + --outputPath${workingDir}/relation --communityMapPath${communityMapPath} @@ -574,14 +572,37 @@ --sourcePath${outputPath}/original/software --resultTableNameeu.dnetlib.dhp.schema.oaf.Software - --outputPath${outputPath}/dump/relation + --outputPath${workingDir}/relation --communityMapPath${communityMapPath} + + + + + + + yarn + cluster + Select valid relations + eu.dnetlib.dhp.oa.graph.dump.subset.SparkSelectValidRelation + dump-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + + --sourcePath${outputPath}/dump + --relationPath${workingDir}/relation + + - - Sub-workflow dump complete failed with error message ${wf:errorMessage()} diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java index 024ff73..9e2b837 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java @@ -1077,4 +1077,35 @@ public class DumpJobTest { .getString(2)); } + @Test + public void testresultNotDumped() throws Exception { + final String sourcePath = getClass() + .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/resultNotDumped.json") + .getPath(); + + final String communityMapPath = getClass() + .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json") + .getPath(); + + SparkDumpEntitiesJob + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", sourcePath, + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication", + "-outputPath", workingDir.toString() + "/result", + "-communityMapPath", communityMapPath + + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/result") + .map(item -> OBJECT_MAPPER.readValue(item, GraphResult.class)); + + Assertions.assertEquals(0, tmp.count()); + + } + } diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java index 7ddc302..eda1556 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java @@ -172,4 +172,37 @@ public class DumpOrganizationProjectDatasourceTest { .println(OBJECT_MAPPER.writeValueAsString(o))); } + @Test + public void dumpDatasourceNotDumpedTest() throws Exception { + final String sourcePath = getClass() + .getResource("/eu/dnetlib/dhp/oa/graph/dump/complete/datasourcenotdumped") + .getPath(); + + SparkDumpEntitiesJob + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", sourcePath, + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Datasource", + "-outputPath", workingDir.toString() + "/dump" + + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/dump") + .map(item -> OBJECT_MAPPER.readValue(item, eu.dnetlib.dhp.oa.model.graph.Datasource.class)); + + org.apache.spark.sql.Dataset verificationDataset = spark + .createDataset(tmp.rdd(), Encoders.bean(eu.dnetlib.dhp.oa.model.graph.Datasource.class)); + + Assertions.assertEquals(1, verificationDataset.count()); + + verificationDataset + .foreach( + (ForeachFunction) o -> System.out + .println(OBJECT_MAPPER.writeValueAsString(o))); + } + } diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/funderresult/SplitPerFunderTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/funderresult/SplitPerFunderTest.java index eed07ae..6d7739c 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/funderresult/SplitPerFunderTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/funderresult/SplitPerFunderTest.java @@ -134,6 +134,8 @@ public class SplitPerFunderTest { .map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class)); Assertions.assertEquals(3, tmp.count()); + tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + // MZOS 1 tmp = sc .textFile(workingDir.toString() + "/split/MZOS") diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/datasourcenotdumped/datasource.json b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/datasourcenotdumped/datasource.json new file mode 100644 index 0000000..a835e66 --- /dev/null +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/complete/datasourcenotdumped/datasource.json @@ -0,0 +1 @@ +{"accessinfopackage":[],"collectedfrom":[{"key":"10|openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"}],"consenttermsofuse":false,"contentpolicies":[{"classid":"Journal articles","classname":"Journal articles","schemeid":"eosc:contentpolicies","schemename":"eosc:contentpolicies"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"datasourcetype":{"classid":"pubsrepository::journal","classname":"Journal","schemeid":"dnet:datasource_typologies","schemename":"dnet:datasource_typologies"},"datasourcetypeui":{"classid":"Journal archive","classname":"Journal archive","schemeid":"dnet:datasource_typologies_ui","schemename":"dnet:datasource_typologies_ui"},"dateofcollection":"2020-07-10","englishname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"value":"Arachnology"},"eoscdatasourcetype":{"classid":"Journal archive","classname":"Journal Archive","schemeid":"dnet:eosc_datasource_types","schemename":"dnet:eosc_datasource_types"},"eosctype":{"classid":"Data Source","classname":"Data Source","schemeid":"","schemename":""},"extraInfo":[],"fulltextdownload":false,"id":"10|issn___print::2d7299a5fd9d7e3db4e6b4c0245fd7c3","journal":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"issnOnline":"2050-9936","issnPrinted":"2050-9928","name":"Arachnology"},"languages":[],"lastupdatetimestamp":1668505479963,"latitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"value":"0.0"},"longitude":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"value":"0.0"},"namespaceprefix":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"value":"jrnl20509928"},"odlanguages":[],"odnumberofitems":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"value":"0.0"},"officialname":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"value":"Arachnology"},"openairecompatibility":{"classid":"hostedBy","classname":"collected from a compatible aggregator","schemeid":"dnet:datasourceCompatibilityLevel","schemename":"dnet:datasourceCompatibilityLevel"},"originalId":["issn___print::2050-9928"],"pid":[],"policies":[],"researchentitytypes":["Literature"],"subjects":[],"thematic":false,"versioncontrol":false,"versioning":{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.900"},"value":false}} \ No newline at end of file diff --git a/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/resultDump/resultNotDumped.json b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/resultDump/resultNotDumped.json new file mode 100644 index 0000000..81eaea7 --- /dev/null +++ b/dump/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/resultDump/resultNotDumped.json @@ -0,0 +1 @@ +{"context": [{"dataInfo": [{"provenanceaction": {"classid": "community:datasource", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "bulktagging", "invisible": false, "trust": "0.8"}], "id": "heritage-science"}], "dataInfo": {"provenanceaction": {"classid": "sysimport:dedup", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "dedup-result-decisiontree-v3", "invisible": false, "trust": "0.8"}, "resourcetype": {"classid": "UNKNOWN", "classname": "UNKNOWN", "schemeid": "dnet:dataCite_resource", "schemename": "dnet:dataCite_resource"}, "pid": [{"qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.3390/geosciences12100362"}], "contributor": [], "oaiprovenance": {"originDescription": {"metadataNamespace": "", "harvestDate": "2022-11-11T12:06:51.264Z", "baseURL": "file%3A%2F%2F%2Fsrv%2Fclaims%2Frecords%2Fpublication%2Fopenaire", "datestamp": "", "altered": true, "identifier": ""}}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "relevantdate": [{"qualifier": {"classid": "created", "classname": "created", "schemeid": "dnet:dataCite_date", "schemename": "dnet:dataCite_date"}, "value": "2022-09-30"}, {"qualifier": {"classid": "published-online", "classname": "published-online", "schemeid": "dnet:dataCite_date", "schemename": "dnet:dataCite_date"}, "value": "2022-09-29"}], "collectedfrom": [{"key": "10|openaire____::c2cdfa5866e03cdd07d313cbc8fb8311", "value": "Multidisciplinary Digital Publishing Institute"}, {"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}, {"key": "10|fairsharing_::cd0f74b5955dc87fd0605745c4b49ee8", "value": "ORCID"}, {"key": "10|infrastruct_::f66f1bd369679b5b077dcdf006089556", "value": "OpenAIRE"}], "id": "50|doi_dedup___::fd21ee82651a8198b88df2ea3cd620bc", "subject": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:subject_classification_typologies", "schemename": "dnet:subject_classification_typologies"}, "value": "lime restoration mortars; global warming potential (environmental impact); hydraulic lime; durability; Lutetian stone; Euville stone; Maya stones"}, {"qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:subject_classification_typologies", "schemename": "dnet:subject_classification_typologies"}, "value": "General Earth and Planetary Sciences"}], "lastupdatetimestamp": 1670006946816, "author": [{"pid": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:actionset", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "orcid_pending", "classname": "Open Researcher and Contributor ID", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "0000-0001-8168-849x"}, {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:entityregistry", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.91", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "orcid", "classname": "Open Researcher and Contributor ID", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "0000-0001-8168-849x"}], "fullname": "Jos\\u00e9 Diaz-Basteris", "surname": "Diaz-Basteris", "name": "Jos\\u00e9", "rank": 1}, {"pid": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:actionset", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "orcid_pending", "classname": "Open Researcher and Contributor ID", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "0000-0001-6379-2891"}, {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:entityregistry", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.91", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "orcid", "classname": "Open Researcher and Contributor ID", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "0000-0001-6379-2891"}], "fullname": "Beatriz Men\\u00e9ndez", "surname": "Men\\u00e9ndez", "name": "Beatriz", "rank": 2}, {"pid": [], "fullname": "Javier Reyes", "surname": "Reyes", "name": "Javier", "rank": 3}, {"pid": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:actionset", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "orcid_pending", "classname": "Open Researcher and Contributor ID", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "0000-0001-9762-3068"}, {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:entityregistry", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.91", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "orcid", "classname": "Open Researcher and Contributor ID", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "0000-0001-9762-3068"}], "fullname": "Julio C. Sacramento Rivero", "surname": "Sacramento Rivero", "name": "Julio C.", "rank": 4}], "instance": [{"refereed": {"classid": "0000", "classname": "UNKNOWN", "schemeid": "dnet:review_levels", "schemename": "dnet:review_levels"}, "hostedby": {"key": "10|doajarticles::8e1bb2f98a85763057afed98376ab4ff", "value": "Geosciences"}, "license": {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "https://creativecommons.org/licenses/by/4.0/"}, "url": ["https://dx.doi.org/10.3390/geosciences12100362"], "pid": [], "distributionlocation": "", "alternateIdentifier": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.3390/geosciences12100362"}], "dateofacceptance": {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "2022-09-29"}, "collectedfrom": {"key": "10|openaire____::c2cdfa5866e03cdd07d313cbc8fb8311", "value": "Multidisciplinary Digital Publishing Institute"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "openAccessRoute": "gold", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "instancetype": {"classid": "0038", "classname": "Other literature type", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}}, {"refereed": {"classid": "0000", "classname": "UNKNOWN", "schemeid": "dnet:review_levels", "schemename": "dnet:review_levels"}, "hostedby": {"key": "10|doajarticles::8e1bb2f98a85763057afed98376ab4ff", "value": "Geosciences"}, "license": {"value": "https://creativecommons.org/licenses/by/4.0/"}, "url": ["https://doi.org/10.3390/geosciences12100362"], "pid": [{"qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.3390/geosciences12100362"}], "dateofacceptance": {"value": "2022-09-29"}, "collectedfrom": {"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "openAccessRoute": "gold", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}}, {"refereed": {"classid": "0000", "classname": "UNKNOWN", "schemeid": "dnet:review_levels", "schemename": "dnet:review_levels"}, "hostedby": {"key": "10|infrastruct_::f66f1bd369679b5b077dcdf006089556", "value": "OpenAIRE"}, "url": ["https://doi.org/10.3390/geosciences12100362"], "pid": [], "distributionlocation": "", "alternateIdentifier": [], "dateofacceptance": {"dataInfo": {"provenanceaction": {"classid": "user:claim", "classname": "Linked by user", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "2022-09-29"}, "collectedfrom": {"key": "10|infrastruct_::f66f1bd369679b5b077dcdf006089556", "value": "OpenAIRE"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "instancetype": {"classid": "0020", "classname": "Other ORP type", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}}], "dateofcollection": "2022-11-11T12:06:51.264Z", "fulltext": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "http://www.mdpi.com/2076-3263/12/10/362/pdf"}], "dateoftransformation": "2022-11-11T12:07:46.017Z", "description": [{"dataInfo": {"provenanceaction": {"classid": "user:claim", "classname": "Linked by user", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "This work proposes sustainability criteria for the selection or design of restoration mortars based on their physical and mechanical properties, durability, price in the French market, and the environmental impact estimated by the global warming potential. A score is assigned to the mortars based on normalized values of their physical and mechanical properties. A total of 24 formulations of restoration mortars were characterized, and their scores were compared. A case study showing the application of the proposed selection method is presented, focused on the restoration of historical monuments in Paris, France, built with Lutetian and Euville stones. In this case, hydraulic lime mortars were the most sustainable options. The application of the method is also projected for global application, as showcased for the restoration of Mayan stones in Southern Mexico."}], "format": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "application/pdf"}], "journal": {"issnPrinted": "2076-3263", "dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "name": "Geosciences; Volume 12; Issue 10; Pages: 362", "edition": "", "vol": "", "sp": "", "iss": "", "issnOnline": "", "ep": "", "issnLinking": ""}, "measures": [], "coverage": [], "externalReference": [], "publisher": {"dataInfo": {"provenanceaction": {"classid": "user:claim", "classname": "Linked by user", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "MDPI AG"}, "eoscifguidelines": [], "language": {"classid": "und", "classname": "Undetermined", "schemeid": "dnet:languages", "schemename": "dnet:languages"}, "resulttype": {"classid": "other", "classname": "other", "schemeid": "dnet:result_typologies", "schemename": "dnet:result_typologies"}, "country": [], "extraInfo": [], "originalId": ["50|multidiscipl::8934c01b1d39b3685d74ff1a5febbf53", "oai:mdpi.com:/2076-3263/12/10/362/", "geosciences12100362", "10.3390/geosciences12100362", "50|doiboost____::fd21ee82651a8198b88df2ea3cd620bc", "", "50|doi_dedup___::fd21ee82651a8198b88df2ea3cd620bc"], "source": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "Geosciences; Volume 12; Issue 10; Pages: 362"}, {"value": "Crossref"}, {"dataInfo": {"provenanceaction": {"classid": "user:claim", "classname": "Linked by user", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "Multidisciplinary Digital Publishing Institute"}], "dateofacceptance": {"dataInfo": {"provenanceaction": {"classid": "user:claim", "classname": "Linked by user", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "2022-09-29"}, "title": [{"dataInfo": {"provenanceaction": {"classid": "user:claim", "classname": "Linked by user", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "main title", "classname": "main title", "schemeid": "dnet:dataCite_title", "schemename": "dnet:dataCite_title"}, "value": "A Selection Method for Restoration Mortars Using Sustainability and Compatibility Criteria"}]} \ No newline at end of file